Support for capturing metadata while doing folly json parsing

Summary: Support for capturing metadata while doing folly json parsing

Reviewed By: yfeldblum

Differential Revision: D13626597

fbshipit-source-id: 8e7a26e23d966240d35c4f6d6104d9334a675a12

folly/dynamic-inl.h

@@ -991,6 +991,11 @@ inline void dynamic::pop_back() {
   arr.pop_back();
 }
 
+inline const dynamic& dynamic::back() const {
+  auto& arr = get<Array>();
+  return arr.back();
+}
+
 //////////////////////////////////////////////////////////////////////
 
 inline dynamic::dynamic(Array&& r) : type_(ARRAY) {

folly/dynamic.h

@@ -710,6 +710,12 @@ struct dynamic : private boost::operators<dynamic> {
    */
   void pop_back();
 
+  /*
+   * Return reference to the last element in an array. If this is not
+   * an array, throws TypeError.
+   */
+  const dynamic& back() const;
+
   /*
    * Get a hash code.  This function is called by a std::hash<>
    * specialization, also.

folly/json.cpp

@@ -218,6 +218,10 @@ struct Input {
     return range_.begin();
   }
 
+  unsigned getLineNum() const {
+    return lineNum_;
+  }
+
   // Parse ahead for as long as the supplied predicate is satisfied,
   // returning a range of what was skipped.
   template <class Predicate>
@@ -362,11 +366,36 @@ class RecursionGuard {
   Input& in_;
 };
 
-dynamic parseValue(Input& in);
+dynamic parseValue(Input& in, json::metadata_map* map);
 std::string parseString(Input& in);
 dynamic parseNumber(Input& in);
 
-dynamic parseObject(Input& in) {
+template <class K>
+void parseObjectKeyValue(
+    Input& in,
+    dynamic& ret,
+    K&& key,
+    json::metadata_map* map) {
+  auto keyLineNumber = in.getLineNum();
+  in.skipWhitespace();
+  in.expect(':');
+  in.skipWhitespace();
+  K tmp;
+  if (map) {
+    tmp = K(key);
+  }
+  auto valueLineNumber = in.getLineNum();
+  ret.insert(std::forward<K>(key), parseValue(in, map));
+  if (map) {
+    auto val = ret.get_ptr(tmp);
+    // We just inserted it, so it should be there!
+    DCHECK(val != nullptr);
+    map->emplace(
+        val, json::parse_metadata{{{keyLineNumber}}, {{valueLineNumber}}});
+  }
+}
+
+dynamic parseObject(Input& in, json::metadata_map* map) {
   DCHECK_EQ(*in, '{');
   ++in;
 
@@ -384,18 +413,12 @@ dynamic parseObject(Input& in) {
     }
     if (*in == '\"') { // string
       auto key = parseString(in);
-      in.skipWhitespace();
-      in.expect(':');
-      in.skipWhitespace();
-      ret.insert(std::move(key), parseValue(in));
+      parseObjectKeyValue(in, ret, std::move(key), map);
     } else if (!in.getOpts().allow_non_string_keys) {
       in.error("expected string for object key name");
     } else {
-      auto key = parseValue(in);
-      in.skipWhitespace();
-      in.expect(':');
-      in.skipWhitespace();
-      ret.insert(std::move(key), parseValue(in));
+      auto key = parseValue(in, map);
+      parseObjectKeyValue(in, ret, std::move(key), map);
     }
 
     in.skipWhitespace();
@@ -410,7 +433,7 @@ dynamic parseObject(Input& in) {
   return ret;
 }
 
-dynamic parseArray(Input& in) {
+dynamic parseArray(Input& in, json::metadata_map* map) {
   DCHECK_EQ(*in, '[');
   ++in;
 
@@ -422,11 +445,15 @@ dynamic parseArray(Input& in) {
     return ret;
   }
 
+  std::vector<uint32_t> lineNumbers;
   for (;;) {
     if (in.getOpts().allow_trailing_comma && *in == ']') {
       break;
     }
-    ret.push_back(parseValue(in));
+    ret.push_back(parseValue(in, map));
+    if (map) {
+      lineNumbers.push_back(in.getLineNum());
+    }
     in.skipWhitespace();
     if (*in != ',') {
       break;
@@ -434,6 +461,11 @@ dynamic parseArray(Input& in) {
     ++in;
     in.skipWhitespace();
   }
+  if (map) {
+    for (size_t i = 0; i < ret.size(); i++) {
+      map->emplace(&ret[i], json::parse_metadata{{{0}}, {{lineNumbers[i]}}});
+    }
+  }
   in.expect(']');
 
   return ret;
@@ -601,14 +633,14 @@ std::string parseString(Input& in) {
   return ret;
 }
 
-dynamic parseValue(Input& in) {
+dynamic parseValue(Input& in, json::metadata_map* map) {
   RecursionGuard guard(in);
 
   in.skipWhitespace();
   // clang-format off
   return
-      *in == '[' ? parseArray(in) :
-      *in == '{' ? parseObject(in) :
+      *in == '[' ? parseArray(in, map) :
+      *in == '{' ? parseObject(in, map) :
       *in == '\"' ? parseString(in) :
       (*in == '-' || (*in >= '0' && *in <= '9')) ? parseNumber(in) :
       in.consume("true") ? true :
@@ -917,6 +949,29 @@ std::string stripComments(StringPiece jsonC) {
 
 //////////////////////////////////////////////////////////////////////
 
+dynamic parseJsonWithMetadata(StringPiece range, json::metadata_map* map) {
+  return parseJsonWithMetadata(range, json::serialization_opts(), map);
+}
+
+dynamic parseJsonWithMetadata(
+    StringPiece range,
+    json::serialization_opts const& opts,
+    json::metadata_map* map) {
+  json::Input in(range, &opts);
+
+  uint32_t n = in.getLineNum();
+  auto ret = parseValue(in, map);
+  if (map) {
+    map->emplace(&ret, json::parse_metadata{{{0}}, {{n}}});
+  }
+
+  in.skipWhitespace();
+  if (in.size() && *in != '\0') {
+    in.error("parsing didn't consume all input");
+  }
+  return ret;
+}
+
 dynamic parseJson(StringPiece range) {
   return parseJson(range, json::serialization_opts());
 }
@@ -924,7 +979,7 @@ dynamic parseJson(StringPiece range) {
 dynamic parseJson(StringPiece range, json::serialization_opts const& opts) {
   json::Input in(range, &opts);
 
-  auto ret = parseValue(in);
+  auto ret = parseValue(in, nullptr);
   in.skipWhitespace();
   if (in.size() && *in != '\0') {
     in.error("parsing didn't consume all input");

folly/json.h

@@ -166,6 +166,23 @@ void escapeString(
  */
 std::string stripComments(StringPiece jsonC);
 
+// may be extened in future to include offset, col, etc.
+struct parse_location {
+  uint32_t line{}; // 0-indexed
+};
+
+// may be extended in future to include end location
+struct parse_range {
+  parse_location begin;
+};
+
+struct parse_metadata {
+  parse_range key_range;
+  parse_range value_range;
+};
+
+using metadata_map = std::unordered_map<dynamic const*, parse_metadata>;
+
 } // namespace json
 
 //////////////////////////////////////////////////////////////////////
@@ -177,6 +194,12 @@ std::string stripComments(StringPiece jsonC);
 dynamic parseJson(StringPiece, json::serialization_opts const&);
 dynamic parseJson(StringPiece);
 
+dynamic parseJsonWithMetadata(StringPiece range, json::metadata_map* map);
+dynamic parseJsonWithMetadata(
+    StringPiece range,
+    json::serialization_opts const& opts,
+    json::metadata_map* map);
+
 /*
  * Serialize a dynamic into a json string.
  */

folly/test/JsonTest.cpp

@@ -23,6 +23,7 @@
 
 using folly::dynamic;
 using folly::parseJson;
+using folly::parseJsonWithMetadata;
 using folly::toJson;
 
 TEST(Json, Unicode) {
@@ -122,6 +123,225 @@ TEST(Json, Parse) {
   EXPECT_EQ(something, expected);
 }
 
+TEST(Json, TestLineNumbers) {
+  // Simple object
+  folly::json::metadata_map map;
+  dynamic val = parseJsonWithMetadata("\n\n{\n\n\"value\":40}", &map);
+  EXPECT_TRUE(val.isObject());
+  auto ov = val.get_ptr("value");
+  EXPECT_TRUE(ov != nullptr);
+  auto it = map.find(ov);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 4);
+  EXPECT_EQ(it->second.value_range.begin.line, 4);
+
+  // check with find() API too
+  auto dv = val.find("value");
+  it = map.find(&dv->second);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 4);
+  EXPECT_EQ(it->second.value_range.begin.line, 4);
+
+  map.clear();
+
+  // One line apart
+  val = parseJsonWithMetadata(
+      "{\"old_value\":40,\n\"changed\":true,\n\"opened\":1.5}", &map);
+
+  EXPECT_TRUE(val.isObject());
+  auto i1 = val.get_ptr("old_value");
+  EXPECT_TRUE(i1 != nullptr);
+  it = map.find(i1);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 0);
+  EXPECT_EQ(it->second.value_range.begin.line, 0);
+  EXPECT_EQ(i1->asInt(), 40);
+
+  auto i2 = val.get_ptr("changed");
+  EXPECT_TRUE(i2 != nullptr);
+  it = map.find(i2);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 1);
+  EXPECT_EQ(it->second.value_range.begin.line, 1);
+  EXPECT_EQ(i2->asBool(), true);
+
+  auto i3 = val.get_ptr("opened");
+  EXPECT_TRUE(i3 != nullptr);
+  it = map.find(i3);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 2);
+  EXPECT_EQ(it->second.value_range.begin.line, 2);
+  EXPECT_EQ(i3->asDouble(), 1.5);
+  map.clear();
+
+  // Multiple lines apart
+  val = parseJsonWithMetadata(
+      "\n{\n\"a\":40,\n\"b\":1.45,\n"
+      "\n\n\"c\":false,\n\n\n\n\n\"d\":\"dval\"\n\n}",
+      &map);
+
+  EXPECT_TRUE(val.isObject());
+
+  i1 = val.get_ptr("a");
+  EXPECT_TRUE(i1 != nullptr);
+  it = map.find(i1);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 2);
+  EXPECT_EQ(it->second.value_range.begin.line, 2);
+  EXPECT_EQ(i1->asInt(), 40);
+
+  i2 = val.get_ptr("b");
+  EXPECT_TRUE(i2 != nullptr);
+  it = map.find(i2);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 3);
+  EXPECT_EQ(it->second.value_range.begin.line, 3);
+  EXPECT_EQ(i2->asDouble(), 1.45);
+
+  i3 = val.get_ptr("c");
+  EXPECT_TRUE(i3 != nullptr);
+  it = map.find(i3);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 6);
+  EXPECT_EQ(it->second.value_range.begin.line, 6);
+  EXPECT_EQ(i3->asBool(), false);
+
+  auto i4 = val.get_ptr("d");
+  EXPECT_TRUE(i4 != nullptr);
+  it = map.find(i4);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 11);
+  EXPECT_EQ(it->second.value_range.begin.line, 11);
+  EXPECT_EQ(i4->asString(), "dval");
+  map.clear();
+
+  // All in the same line
+  val = parseJsonWithMetadata("{\"x\":40,\"y\":true,\"z\":3.33}", &map);
+  EXPECT_TRUE(val.isObject());
+
+  i1 = val.get_ptr("x");
+  EXPECT_TRUE(i1 != nullptr);
+  it = map.find(i1);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 0);
+  EXPECT_EQ(it->second.value_range.begin.line, 0);
+  EXPECT_EQ(i1->asInt(), 40);
+
+  i2 = val.get_ptr("y");
+  EXPECT_TRUE(i2 != nullptr);
+  it = map.find(i2);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 0);
+  EXPECT_EQ(it->second.value_range.begin.line, 0);
+  EXPECT_EQ(i2->asBool(), true);
+
+  i3 = val.get_ptr("z");
+  EXPECT_TRUE(i3 != nullptr);
+  it = map.find(i3);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 0);
+  EXPECT_EQ(it->second.value_range.begin.line, 0);
+  map.clear();
+
+  // Key and value in different numbers
+  val =
+      parseJsonWithMetadata("{\"x\":\n70,\"y\":\n\n80,\n\"z\":\n\n\n33}", &map);
+  EXPECT_TRUE(val.isObject());
+
+  i1 = val.get_ptr("x");
+  EXPECT_TRUE(i1 != nullptr);
+  it = map.find(i1);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 0);
+  EXPECT_EQ(it->second.value_range.begin.line, 1);
+  EXPECT_EQ(i1->asInt(), 70);
+
+  i2 = val.get_ptr("y");
+  EXPECT_TRUE(i2 != nullptr);
+  it = map.find(i2);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 1);
+  EXPECT_EQ(it->second.value_range.begin.line, 3);
+  EXPECT_EQ(i2->asInt(), 80);
+
+  i3 = val.get_ptr("z");
+  EXPECT_TRUE(i3 != nullptr);
+  it = map.find(i3);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 4);
+  EXPECT_EQ(it->second.value_range.begin.line, 7);
+  EXPECT_EQ(i3->asInt(), 33);
+  map.clear();
+
+  // With Arrays
+  val = parseJsonWithMetadata(
+      "{\"x\":\n[10, 20],\n\"y\":\n\n[80,\n90,\n100]}", &map);
+  EXPECT_TRUE(val.isObject());
+
+  i1 = val.get_ptr("x");
+  EXPECT_TRUE(i1 != nullptr);
+  it = map.find(i1);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 0);
+  EXPECT_EQ(it->second.value_range.begin.line, 1);
+
+  int i = 1;
+  for (auto arr_it = i1->begin(); arr_it != i1->end(); arr_it++, i++) {
+    auto arr_it_md = map.find(&*arr_it);
+    EXPECT_TRUE(arr_it_md != map.end());
+    EXPECT_EQ(arr_it_md->second.value_range.begin.line, 1);
+    EXPECT_EQ(arr_it->asInt(), 10 * i);
+  }
+
+  i2 = val.get_ptr("y");
+  EXPECT_TRUE(i2 != nullptr);
+  it = map.find(i2);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 2);
+  EXPECT_EQ(it->second.value_range.begin.line, 4);
+
+  i = 8;
+  int ln = 4;
+  for (auto arr_it = i2->begin(); arr_it != i2->end(); arr_it++, i++, ln++) {
+    auto arr_it_md = map.find(&*arr_it);
+    EXPECT_TRUE(arr_it_md != map.end());
+    EXPECT_EQ(arr_it_md->second.value_range.begin.line, ln);
+    EXPECT_EQ(arr_it->asInt(), 10 * i);
+  }
+  map.clear();
+
+  // With nested objects
+  val = parseJsonWithMetadata(
+      "{\"a1\":{\n\"a2\":{\n\"a3\":{\n\"a4\":4}}}}", &map);
+  EXPECT_TRUE(val.isObject());
+  i1 = val.get_ptr("a1");
+  EXPECT_TRUE(i1 != nullptr);
+  it = map.find(i1);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 0);
+  EXPECT_EQ(it->second.value_range.begin.line, 0);
+  i2 = i1->get_ptr("a2");
+  EXPECT_TRUE(i2 != nullptr);
+  it = map.find(i2);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 1);
+  EXPECT_EQ(it->second.value_range.begin.line, 1);
+  i3 = i2->get_ptr("a3");
+  EXPECT_TRUE(i3 != nullptr);
+  it = map.find(i3);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 2);
+  EXPECT_EQ(it->second.value_range.begin.line, 2);
+  i4 = i3->get_ptr("a4");
+  EXPECT_TRUE(i4 != nullptr);
+  it = map.find(i4);
+  EXPECT_TRUE(it != map.end());
+  EXPECT_EQ(it->second.key_range.begin.line, 3);
+  EXPECT_EQ(it->second.value_range.begin.line, 3);
+  EXPECT_EQ(i4->asInt(), 4);
+  map.clear();
+}
+
 TEST(Json, ParseTrailingComma) {
   folly::json::serialization_opts on, off;
   on.allow_trailing_comma = true;