From 2428da22d19e8dfb66b30d41df15ea0e38731868 Mon Sep 17 00:00:00 2001
From: David Malcolm <dmalcolm@redhat.com>
Date: Tue, 23 May 2017 23:10:06 -0400
Subject: [PATCH 02/38] FIXME: fixes to json.c/h not in other branches
 (unicode-handling, literals, accessors)

---
 gcc/json.c | 417 +++++++++++++++++++++++++++++++++++++++++++++++++++++++------
 gcc/json.h |  11 ++
 2 files changed, 387 insertions(+), 41 deletions(-)

diff --git a/gcc/json.c b/gcc/json.c
index 4d3e8fa..73bf95f 100644
--- a/gcc/json.c
+++ b/gcc/json.c
@@ -22,6 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "coretypes.h"
 #include "json.h"
 #include "pretty-print.h"
+#include "math.h"
 #include "selftest.h"
 
 using namespace json;
@@ -96,6 +97,66 @@ value::as_string () const
   return static_cast <const string *> (this);
 }
 
+/* Attempt to get the value of a key/value pair from this value
+   as if THIS value were an object.
+
+   If THIS is not a json::object, return write an error message to OUT_ERR
+   (which must be freed by the caller) and return false.
+
+   Otherwise write the value ptr (possibly NULL) to OUT_VALUE and
+   return true.  */
+
+bool
+value::get_optional_value_by_key (const char *name, const value *&out_value,
+				  char *&out_err) const
+{
+  const json::object *obj = as_object ();
+  if (!obj)
+    {
+      out_err = xstrdup ("not an object");
+      return false;
+    }
+  out_value = obj->get (name);
+  return true;
+}
+
+/* Attempt to get a string value of a key/value pair from this value
+   as if THIS value were an object.
+
+   If THIS is a json::object, and KEY is either not present, is a string,
+   or is the "null" JSON literal, then return true, and write to OUT_VALUE.
+   If a string, then the ptr is written to OUT_VALUE, otherwise NULL
+   is written to OUT_VALUE.
+
+   If THIS is not a json::object, or KEY is not a string/"null",
+   return false and write an error message to OUT_ERR
+   (which must be freed by the caller).  */
+
+bool
+value::get_optional_string_by_key (const char *name, const char *&out_value,
+				   char *&out_err) const
+{
+  const json::value *v;
+  if (!get_optional_value_by_key (name, v, out_err))
+    return false;
+  if (v && v->get_kind () != JSON_NULL)
+    {
+      const json::string *s = v->as_string ();
+      if (!s)
+	{
+	  out_err = xasprintf ("not a string: \"%s\"", name);
+	  return false;
+	}
+      out_value = s->get_string ();
+      return true;
+    }
+  else
+    {
+      out_value = NULL;
+      return true;
+    }
+}
+
 /* Attempt to get lookup the value of a key/value pair from this value
    as if this value were an object.
 
@@ -165,6 +226,26 @@ value::get_string_by_key (const char *name, const char *&out_value,
   return true;
 }
 
+/* As value::get_value_by_key, but the value must be an array;
+   if successful, write it as a json::array * to OUT_VALUE.  */
+
+bool
+value::get_array_by_key (const char *name, const array *&out_value,
+			 char *&out_err) const
+{
+  const json::value *v;
+  if (!get_value_by_key (name, v, out_err))
+    return false;
+  const json::array *arr = v->as_array ();
+  if (!arr)
+    {
+      out_err = xasprintf ("not an array: \"%s\"", name);
+      return false;
+    }
+  out_value = arr;
+  return true;
+}
+
 /* class json::object, a subclass of json::value, representing
    an unordered collection of key/value pairs.  */
 
@@ -224,6 +305,20 @@ object::get (const char *key) const
   return NULL;
 }
 
+/* As object::get (KEY), but return NULL if the value of the key
+   is the "null" JSON literal.  */
+
+value *
+object::get_if_nonnull (const char *key) const
+{
+  value *result = get (key);
+  if (!result)
+    return NULL;
+  if (result->get_kind () == JSON_NULL)
+    return NULL;
+  return result;
+}
+
 /* Set the json::value * for KEY, taking ownership of VALUE
    (and taking a copy of KEY).  */
 
@@ -283,7 +378,17 @@ array::clone () const
 void
 number::print (pretty_printer *pp) const
 {
-  pp_printf (pp, "%i", (int)m_value); // FIXME
+  pp_printf (pp, "%i", (int)m_value);
+  double remaining = fabs (m_value - (int)m_value);
+  // FIXME: ensure termination
+  if (remaining)
+    pp_character (pp, '.');
+  while (remaining > 0.0001) // FIXME
+    {
+      remaining *= 10;
+      pp_printf (pp, "%i", (int)remaining);
+      remaining = remaining - (int)remaining;
+    }
 }
 
 /* Implementation of json::value::clone for json::number.  */
@@ -305,14 +410,27 @@ string::print (pretty_printer *pp) const
       char ch = *ptr;
       switch (ch)
 	{
+	case '"':
+	  pp_string (pp, "\\\"");
+	  break;
 	case '\\':
 	  pp_string (pp, "\\n");
 	  break;
-	case '"':
-	  pp_string (pp, "\\\"");
+	case '\b':
+	  pp_string (pp, "\\b");
+	  break;
+	case '\f':
+	  pp_string (pp, "\\f");
+	  break;
+	case '\n':
+	  pp_string (pp, "\\n");
+	  break;
+	case '\r':
+	  pp_string (pp, "\\r");
+	  break;
+	case '\t':
+	  pp_string (pp, "\\t");
 	  break;
-
-	  // FIXME: handle other escapes
 
 	default:
 	  pp_character (pp, ch);
@@ -426,7 +544,7 @@ struct token
 
   union
   {
-    /* Value for TOK_STRING.  */
+    /* Value for TOK_ERROR and TOK_STRING.  */
     char *string;
 
     /* Value for TOK_NUMBER.  */
@@ -440,7 +558,7 @@ class lexer
 {
  public:
   lexer ();
-  void add_utf8 (size_t length, const char *utf8_buf);
+  bool add_utf8 (size_t length, const char *utf8_buf, char **err_out);
 
   const token *peek ();
   void consume ();
@@ -452,7 +570,7 @@ class lexer
   void lex_token (token *out);
   void lex_string (token *out);
   void lex_number (token *out, unichar first_char);
-  bool rest_of_literal_p (const char *suffix) const;
+  bool rest_of_literal (const char *suffix);
 
  private:
   auto_vec<unichar> m_buffer;
@@ -469,7 +587,7 @@ class parser
 {
  public:
   parser (char **err_out);
-  void add_utf8 (size_t length, const char *utf8_buf);
+  bool add_utf8 (size_t length, const char *utf8_buf, char **err_out);
   value *parse_value ();
   object *parse_object ();
   array *parse_array ();
@@ -527,7 +645,8 @@ lexer::consume ()
       fprintf (stderr, "\n");
     }
 
-  if (m_next_tokens[0].id == TOK_STRING)
+  if (m_next_tokens[0].id == TOK_ERROR
+      || m_next_tokens[0].id == TOK_STRING)
     free (m_next_tokens[0].u.string);
 
   m_num_next_tokens--;
@@ -535,16 +654,81 @@ lexer::consume ()
 	   sizeof (token) * m_num_next_tokens);
 }
 
-/* Add LENGTH bytes of UTF-8 encoded text from UTF8_BUF to this lexer'
+/* Add LENGTH bytes of UTF-8 encoded text from UTF8_BUF to this lexer's
    buffer.  */
 
-void
-lexer::add_utf8 (size_t length, const char *utf8_buf)
+bool
+lexer::add_utf8 (size_t length, const char *utf8_buf, char **err_out)
 {
-  /* FIXME: this blithely ignores the niceties of UTF-8 and simply pushes the
-     bytes into the buffer.  */
-  for (size_t i = 0; i < length; i++)
-    m_buffer.safe_push (utf8_buf[i]);
+  /* FIXME: adapted from charset.c:one_utf8_to_cppchar.  */
+  static const uchar masks[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
+  static const uchar patns[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
+
+  const uchar *inbuf = (const unsigned char *) (utf8_buf);
+  const uchar **inbufp = &inbuf;
+  size_t *inbytesleftp = &length;
+
+  while (length > 0)
+    {
+      unichar c;
+      const uchar *inbuf = *inbufp;
+      size_t nbytes, i;
+
+      c = *inbuf;
+      if (c < 0x80)
+	{
+	  m_buffer.safe_push (c);
+	  *inbytesleftp -= 1;
+	  *inbufp += 1;
+	  continue;
+	}
+
+      /* The number of leading 1-bits in the first byte indicates how many
+	 bytes follow.  */
+      for (nbytes = 2; nbytes < 7; nbytes++)
+	if ((c & ~masks[nbytes-1]) == patns[nbytes-1])
+	  goto found;
+      *err_out = xstrdup ("ill-formed UTF-8 sequence");
+      return false;
+    found:
+
+      if (*inbytesleftp < nbytes)
+	return false;
+
+      c = (c & masks[nbytes-1]);
+      inbuf++;
+      for (i = 1; i < nbytes; i++)
+	{
+	  unichar n = *inbuf++;
+	  if ((n & 0xC0) != 0x80)
+	    return false;
+	  c = ((c << 6) + (n & 0x3F));
+	}
+
+      /* Make sure the shortest possible encoding was used.  */
+      if ((   c <=      0x7F && nbytes > 1)
+	  || (c <=     0x7FF && nbytes > 2)
+	  || (c <=    0xFFFF && nbytes > 3)
+	  || (c <=  0x1FFFFF && nbytes > 4)
+	  || (c <= 0x3FFFFFF && nbytes > 5))
+	{
+	  *err_out = xstrdup ("ill-formed UTF-8:"
+			      " shortest possible encoding not used");
+	  return false;
+	}
+
+      /* Make sure the character is valid.  */
+      if (c > 0x7FFFFFFF || (c >= 0xD800 && c <= 0xDFFF))
+	{
+	  *err_out = xstrdup ("ill-formed UTF-8: invalid character");
+	  return false;
+	}
+
+      m_buffer.safe_push (c);
+      *inbufp = inbuf;
+      *inbytesleftp -= nbytes;
+    }
+  return true;
 }
 
 /* Attempt to get the next unicode character from this lexer's buffer.
@@ -579,7 +763,7 @@ lexer::dump_token (FILE *outf, const token *tok)
   switch (tok->id)
     {
     case TOK_ERROR:
-      fprintf (outf, "TOK_ERROR");
+      fprintf (outf, "TOK_ERROR (\"%s\")", tok->u.string);
       break;
 
     case TOK_EOF:
@@ -652,7 +836,10 @@ lexer::lex_token (token *out)
 	  out->id = TOK_EOF;
 	  return;
 	}
-      if (next_char != ' ' && next_char != '\t')
+      if (next_char != ' '
+	  && next_char != '\t'
+	  && next_char != '\n'
+	  && next_char != '\r')
 	break;
     }
 
@@ -702,7 +889,7 @@ lexer::lex_token (token *out)
 
     case 't':
       /* Handle literal "true".  */
-      if (rest_of_literal_p ("rue"))
+      if (rest_of_literal ("rue"))
 	{
 	  out->id = TOK_TRUE;
 	  break;
@@ -712,7 +899,7 @@ lexer::lex_token (token *out)
 
     case 'f':
       /* Handle literal "false".  */
-      if (rest_of_literal_p ("alse"))
+      if (rest_of_literal ("alse"))
 	{
 	  out->id = TOK_FALSE;
 	  break;
@@ -722,7 +909,7 @@ lexer::lex_token (token *out)
 
     case 'n':
       /* Handle literal "null".  */
-      if (rest_of_literal_p ("ull"))
+      if (rest_of_literal ("ull"))
 	{
 	  out->id = TOK_NULL;
 	  break;
@@ -733,6 +920,7 @@ lexer::lex_token (token *out)
     err:
     default:
       out->id = TOK_ERROR;
+      out->u.string = xasprintf ("unexpected character: %c", next_char);
       break;
     }
 }
@@ -752,6 +940,7 @@ lexer::lex_string (token *out)
       if (!get_char (uc))
 	{
 	  out->id = TOK_ERROR;
+	  out->u.string = xasprintf ("EOF within string");
 	  return;
 	}
       switch (uc)
@@ -765,19 +954,72 @@ lexer::lex_string (token *out)
 	    if (!get_char (next_char))
 	      {
 		out->id = TOK_ERROR;
+		out->u.string = xasprintf ("EOF within string");;
 		return;
 	      }
 	    switch (next_char)
 	      {
-	      case '\\':
 	      case '"':
+	      case '\\':
+	      case '/':
 		content.safe_push (next_char);
 		break;
 
+	      case 'b':
+		content.safe_push ('\b');
+		break;
+
+	      case 'f':
+		content.safe_push ('\f');
+		break;
+
+	      case 'n':
+		content.safe_push ('\n');
+		break;
+
+	      case 'r':
+		content.safe_push ('\r');
+		break;
+
+	      case 't':
+		content.safe_push ('\t');
+		break;
+
+	      case 'u':
+		{
+		  unichar result = 0;
+		  for (int i = 0; i < 4; i++)
+		    {
+		      unichar hexdigit;
+		      if (!get_char (hexdigit))
+			{
+			  out->id = TOK_ERROR;
+			  out->u.string = xasprintf ("EOF within string");
+			  return;
+			}
+		      result <<= 4;
+		      if (hexdigit >= '0' && hexdigit <= '9')
+			result += hexdigit - '0';
+		      else if (hexdigit >= 'a' && hexdigit <= 'f')
+			result += (hexdigit - 'a') + 10;
+		      else if (hexdigit >= 'A' && hexdigit <= 'F')
+			result += (hexdigit - 'A') + 10;
+		      else
+			{
+			  out->id = TOK_ERROR;
+			  out->u.string = xasprintf ("bogus hex char");
+			  return;
+			}
+		    }
+		  content.safe_push (result);
+		}
+		break;
+
 		// FIXME: implement other chars
 
 	      default:
 		out->id = TOK_ERROR;
+		out->u.string = xasprintf ("unrecognized escape char");
 		return;
 	      }
 	  }
@@ -791,11 +1033,39 @@ lexer::lex_string (token *out)
 
   out->id = TOK_STRING;
 
-  // FIXME: this assume it's all Latin-1
-  out->u.string = XNEWVEC (char, content.length () + 1);
+  auto_vec<char> utf8_buf;
+  // FIXME: adapted from libcpp/charset.c:one_cppchar_to_utf8
   for (unsigned i = 0; i < content.length (); i++)
-    out->u.string[i] = content[i];
-  out->u.string[content.length ()] = '\0';
+    {
+      static const uchar masks[6] =  { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
+      static const uchar limits[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
+      size_t nbytes;
+      uchar buf[6], *p = &buf[6];
+      unichar c = content[i];
+
+      nbytes = 1;
+      if (c < 0x80)
+	*--p = c;
+      else
+	{
+	  do
+	    {
+	      *--p = ((c & 0x3F) | 0x80);
+	      c >>= 6;
+	      nbytes++;
+	    }
+	  while (c >= 0x3F || (c & limits[nbytes-1]));
+	  *--p = (c | masks[nbytes-1]);
+	}
+
+      while (p < &buf[6])
+	utf8_buf.safe_push (*p++);
+    }
+
+  out->u.string = XNEWVEC (char, utf8_buf.length () + 1);
+  for (unsigned i = 0; i < utf8_buf.length (); i++)
+    out->u.string[i] = utf8_buf[i];
+  out->u.string[utf8_buf.length ()] = '\0';
 
   // FIXME: leaks?  have a json_context do the allocation
 }
@@ -832,9 +1102,33 @@ lexer::lex_number (token *out, unichar first_char)
 	  unget_char ();
 	  break;
 	}
-      // FIXME '.' and 'e'/'E'
     }
 
+  unichar next_char;
+  if (get_char (next_char))
+    {
+      if (next_char == '.')
+	{
+	  /* Parse decimal digits.  */
+	  // FIXME: does this lose too much precision?
+	  double digit_factor = 0.1;
+	  while (get_char (next_char))
+	    {
+	      if (!ISDIGIT (next_char))
+		{
+		  unget_char ();
+		  break;
+		}
+	      value += (next_char - '0') * digit_factor;
+	      digit_factor *= 0.1;
+	    }
+	}
+      else
+	unget_char ();
+    }
+
+  /* TODO: parse 'e' and 'E'.  */
+
   if (negate)
     value = -value;
 
@@ -842,18 +1136,24 @@ lexer::lex_number (token *out, unichar first_char)
   out->u.number = value;
 }
 
-/* Determine if the next characters to be lexed match SUFFIX.  */
+/* Determine if the next characters to be lexed match SUFFIX.
+   SUFFIX must be pure ASCII.
+   If so, consume the characters and return true.
+   Otherwise, return false.  */
 
 bool
-lexer::rest_of_literal_p (const char *suffix) const
+lexer::rest_of_literal (const char *suffix)
 {
   int suffix_idx = 0;
   int buf_idx = m_next_char_idx;
   while (1)
     {
       if (suffix[suffix_idx] == '\0')
-	return true;
-      /* FIXME: this assumes ASCII.  */
+	{
+	  m_next_char_idx += suffix_idx;
+	  return true;
+	}
+      /* This assumes that suffix is ASCII.  */
       if (m_buffer[buf_idx] != (unichar)suffix[suffix_idx])
 	return false;
       buf_idx++;
@@ -874,10 +1174,10 @@ parser::parser (char **err_out)
 /* Add LENGTH bytes of UTF-8 encoded text from UTF8_BUF to this parser's
    lexer's buffer.  */
 
-void
-parser::add_utf8 (size_t length, const char *utf8_buf)
+bool
+parser::add_utf8 (size_t length, const char *utf8_buf, char **err_out)
 {
-  m_lexer.add_utf8 (length, utf8_buf);
+  return m_lexer.add_utf8 (length, utf8_buf, err_out);
 }
 
 /* FIXME.  */
@@ -1036,8 +1336,14 @@ parser::require (enum token_id tok_id)
 {
   const token *tok = m_lexer.peek ();
   if (tok->id != tok_id)
-    error_at (tok->index, "expected %s; got %s", token_id_name[tok_id],
-	      token_id_name[tok->id]);
+    {
+      if (tok->id == TOK_ERROR)
+	error_at (tok->index, "expected %s; got bad token: %s",
+		  token_id_name[tok_id], tok->u.string);
+      else
+	error_at (tok->index, "expected %s; got %s", token_id_name[tok_id],
+		  token_id_name[tok->id]);
+    }
   m_lexer.consume ();
 }
 
@@ -1082,7 +1388,8 @@ json::parse_utf8_string (size_t length, const char *utf8_buf,
   gcc_assert (*err_out == NULL);
 
   parser p (err_out);
-  p.add_utf8 (length, utf8_buf);
+  if (!p.add_utf8 (length, utf8_buf, err_out))
+    return NULL;
   value *result = p.parse_value ();
   if (p.seen_error_p ())
     {
@@ -1180,6 +1487,18 @@ test_parse_string ()
   /* Result of dump should be UTF-8 encoded, with quotes.  */
   assert_to_str_eq (mojibake, jv);
   delete jv;
+
+  /* Test of \u-escaped unicode.  This is "mojibake" again, as above.  */
+  const char *escaped_unicode = "\"\\u6587\\u5b57\\u5316\\u3051\"";
+  jv = parse_utf8_string (escaped_unicode, &err);
+  ASSERT_EQ (NULL, err);
+  ASSERT_EQ (JSON_STRING, jv->get_kind ());
+  /* Result of get_string should be UTF-8 encoded, without quotes.  */
+  ASSERT_STREQ ("\346\226\207" "\345\255\227" "\345\214\226" "\343\201\221",
+		((json::string *)jv)->get_string ());
+  /* Result of dump should be UTF-8 encoded, with quotes.  */
+  assert_to_str_eq (mojibake, jv);
+  delete jv;
 }
 
 /* FIXME.  */
@@ -1199,6 +1518,22 @@ test_parse_number ()
   ASSERT_EQ (JSON_NUMBER, clone->get_kind ());
   delete clone;
   delete jv;
+
+  /* Negative number.  */
+  jv = parse_utf8_string ("-17", &err);
+  ASSERT_EQ (NULL, err);
+  ASSERT_EQ (JSON_NUMBER, jv->get_kind ());
+  ASSERT_EQ (-17.0, ((json::number *)jv)->get ());
+  assert_to_str_eq ("-17", jv);
+  delete jv;
+
+  /* Decimal.  */
+  jv = parse_utf8_string ("3.141", &err);
+  ASSERT_EQ (NULL, err);
+  ASSERT_EQ (JSON_NUMBER, jv->get_kind ());
+  ASSERT_EQ (3.141, ((json::number *)jv)->get ());
+  assert_to_str_eq ("3.141", jv);
+  delete jv;
 }
 
 /* FIXME.  */
@@ -1245,7 +1580,7 @@ test_parse_object ()
 {
   char *err = NULL;
   json::value *jv
-    = parse_utf8_string ("{\"foo\": \"bar\", \"baz\": [42, 43]}", &err);
+    = parse_utf8_string ("{\"foo\": \"bar\", \"baz\": [42, null]}", &err);
   ASSERT_EQ (NULL, err);
   ASSERT_TRUE (jv != NULL);
   ASSERT_EQ (JSON_OBJECT, jv->get_kind ());
@@ -1262,8 +1597,8 @@ test_parse_object ()
 
   json::array *baz_array = (json::array *)baz_value;
   ASSERT_EQ (2, baz_array->get_length ());
-
-  // etc, verify numbers
+  ASSERT_EQ (42, baz_array->get (0)->as_number ()->get ());
+  ASSERT_EQ (JSON_NULL, baz_array->get (1)->get_kind ());
 
   // TODO: error-handling
   // TODO: partial document
diff --git a/gcc/json.h b/gcc/json.h
index b6bdb13..e97d384 100644
--- a/gcc/json.h
+++ b/gcc/json.h
@@ -98,6 +98,15 @@ class value
   bool get_int_by_key (const char *name, int &out_value, char *&out_err) const;
   bool get_string_by_key (const char *name, const char *&out_value,
 			  char *&out_err) const;
+  bool get_array_by_key (const char *name, const array *&out_value,
+			 char *&out_err) const;
+
+  /* As above, but the key is optional.  THIS must still be an object,
+     though.  */
+  bool get_optional_value_by_key (const char *name, const value *&out_value,
+				  char *&out_err) const;
+  bool get_optional_string_by_key (const char *name, const char *&out_value,
+				   char *&out_err) const;
 };
 
 /* Subclass of value for objects: an unordered collection of
@@ -113,6 +122,8 @@ class object : public value
   value *clone () const FINAL OVERRIDE;
 
   value *get (const char *key) const;
+  value *get_if_nonnull (const char *key) const;
+
   void set (const char *key, value *v);
 
  private:
-- 
1.8.5.3

