Add JSONPointer support

Summary:
Allow retrieving sub-tree from folly::dynamic using JSON pointer syntax (RFC6901). This adds new overload for the `get_ptr` method, retrieving raw pointer to a sub-tree, or returning `nullptr` if element is not found.

The `folly::dynamic` implementation traverses the path specified by JSON pointer string dynamically every time, since the underlying object may change, and JSON pointer traversal depends on the underlying object structure. E.g. "123" could be a key name for `dynamic::object` and index in `dynamic::array`.

Reviewed By: yfeldblum

Differential Revision: D6790515

fbshipit-source-id: bb6ea10acb83673e87721cf1e5a02506b44bb273

CMakeLists.txt

@@ -661,6 +661,7 @@ if (BUILD_TESTS)
         SOURCES IndexedMemPoolTest.cpp
       # MSVC Preprocessor stringizing raw string literals bug
       #TEST json_test SOURCES JsonTest.cpp
+      TEST json_pointer_test SOURCES json_pointer_test.cpp
       TEST json_other_test
         CONTENT_DIR json_test_data/
         SOURCES

folly/Makefile.am

@@ -323,6 +323,7 @@ nobase_follyinclude_HEADERS = \
 	io/async/test/UndelayedDestruction.h \
 	io/async/test/Util.h \
 	json.h \
+	json_pointer.h \
 	lang/Align.h \
 	lang/Assume.h \
 	lang/Bits.h \
@@ -576,6 +577,7 @@ libfolly_la_SOURCES = \
 	io/async/ssl/OpenSSLUtils.cpp \
 	io/async/ssl/SSLErrors.cpp \
 	json.cpp \
+	json_pointer.cpp \
 	lang/Assume.cpp \
 	lang/ColdClass.cpp \
 	lang/SafeAssert.cpp \

folly/dynamic-inl.h

@@ -586,6 +586,11 @@ inline dynamic* dynamic::get_ptr(dynamic const& idx) & {
   return const_cast<dynamic*>(const_cast<dynamic const*>(this)->get_ptr(idx));
 }
 
+inline dynamic* dynamic::get_ptr(json_pointer const& jsonPtr) & {
+  return const_cast<dynamic*>(
+      const_cast<dynamic const*>(this)->get_ptr(jsonPtr));
+}
+
 inline dynamic& dynamic::at(dynamic const& idx) & {
   return const_cast<dynamic&>(const_cast<dynamic const*>(this)->at(idx));
 }

folly/dynamic.cpp

@@ -349,6 +349,48 @@ dynamic dynamic::merge_diff(const dynamic& source, const dynamic& target) {
   return diff;
 }
 
+const dynamic* dynamic::get_ptr(json_pointer const& jsonPtr) const& {
+  auto const& tokens = jsonPtr.tokens();
+  if (tokens.empty()) {
+    return this;
+  }
+  dynamic const* dyn = this;
+  for (auto const& token : tokens) {
+    if (!dyn) {
+      return nullptr;
+    }
+    // special case of parsing "/": lookup key with empty name
+    if (token.empty()) {
+      if (dyn->isObject()) {
+        dyn = dyn->get_ptr("");
+        continue;
+      }
+      throwTypeError_("object", dyn->type());
+    }
+    if (auto* parray = dyn->get_nothrow<dynamic::Array>()) {
+      if (token.size() > 1 && token.at(0) == '0') {
+        throw std::invalid_argument(
+            "Leading zero not allowed when indexing arrays");
+      }
+      // special case, always return non-existent
+      if (token.size() == 1 && token.at(0) == '-') {
+        dyn = nullptr;
+        continue;
+      }
+      auto const idx = folly::to<size_t>(token);
+      dyn = idx < parray->size() ? &(*parray)[idx] : nullptr;
+      continue;
+    }
+    if (auto* pobject = dyn->get_nothrow<dynamic::ObjectImpl>()) {
+      auto const it = pobject->find(token);
+      dyn = it != pobject->end() ? &it->second : nullptr;
+      continue;
+    }
+    throwTypeError_("object/array", dyn->type());
+  }
+  return dyn;
+}
+
 //////////////////////////////////////////////////////////////////////
 
 } // namespace folly

folly/dynamic.h

@@ -65,6 +65,7 @@
 
 #include <folly/Range.h>
 #include <folly/Traits.h>
+#include <folly/json_pointer.h>
 
 namespace folly {
 
@@ -374,6 +375,16 @@ struct dynamic : private boost::operators<dynamic> {
   dynamic&       at(dynamic const&) &;
   dynamic&&      at(dynamic const&) &&;
 
+  /*
+   * Locate element using JSON pointer, per RFC 6901. Returns nullptr if
+   * element could not be located. Throws if pointer does not match the
+   * shape of the document, e.g. uses string to index in array.
+   */
+  const dynamic* get_ptr(json_pointer const&) const&;
+  dynamic* get_ptr(json_pointer const&) &;
+  const dynamic* get_ptr(json_pointer const&) const&& = delete;
+  dynamic* get_ptr(json_pointer const&) && = delete;
+
   /*
    * Like 'at', above, except it returns either a pointer to the contained
    * object or nullptr if it wasn't found. This allows a key to be tested for

folly/json_pointer.cpp

+/*
+ * Copyright 2011-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/json_pointer.h>
+
+#include <folly/String.h>
+
+namespace folly {
+
+// static, public
+Expected<json_pointer, json_pointer::parse_error> json_pointer::try_parse(
+    StringPiece const str) {
+  // pointer describes complete document
+  if (str.empty()) {
+    return json_pointer{};
+  }
+
+  if (str.at(0) != '/') {
+    return makeUnexpected(parse_error::INVALID_FIRST_CHARACTER);
+  }
+
+  std::vector<std::string> tokens;
+  splitTo<std::string>("/", str, std::inserter(tokens, tokens.begin()));
+  tokens.erase(tokens.begin());
+
+  for (auto& token : tokens) {
+    if (!unescape(token)) {
+      return makeUnexpected(parse_error::INVALID_ESCAPE_SEQUENCE);
+    }
+  }
+
+  return json_pointer(std::move(tokens));
+}
+
+// static, public
+json_pointer json_pointer::parse(StringPiece const str) {
+  auto res = try_parse(str);
+  if (res.hasValue()) {
+    return std::move(res.value());
+  }
+  switch (res.error()) {
+    case parse_error::INVALID_FIRST_CHARACTER:
+      throw json_pointer::parse_exception(
+          "non-empty JSON pointer string does not start with '/'");
+    case parse_error::INVALID_ESCAPE_SEQUENCE:
+      throw json_pointer::parse_exception(
+          "Invalid escape sequence in JSON pointer string");
+    default:
+      assume_unreachable();
+  }
+}
+
+std::vector<std::string> const& json_pointer::tokens() const {
+  return tokens_;
+}
+
+// private
+json_pointer::json_pointer(std::vector<std::string> tokens) noexcept
+    : tokens_{std::move(tokens)} {}
+
+// private, static
+bool json_pointer::unescape(std::string& str) {
+  char const* end = &str[str.size()];
+  char* out = &str.front();
+  char const* decode = out;
+  while (decode < end) {
+    if (*decode != '~') {
+      *out++ = *decode++;
+      continue;
+    }
+    if (decode + 1 == end) {
+      return false;
+    }
+    switch (decode[1]) {
+      case '1':
+        *out++ = '/';
+        break;
+      case '0':
+        *out++ = '~';
+        break;
+      default:
+        return false;
+    }
+    decode += 2;
+  }
+  str.resize(out - &str.front());
+  return true;
+}
+
+} // namespace folly

folly/json_pointer.h

+/*
+ * Copyright 2011-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include <folly/Expected.h>
+#include <folly/Range.h>
+
+namespace folly {
+
+/*
+ * json_pointer
+ *
+ * As described in RFC 6901 "JSON Pointer".
+ *
+ * Implements parsing. Traversal using the pointer over data structures must be
+ * implemented separately.
+ */
+class json_pointer {
+ public:
+  enum class parse_error {
+    INVALID_FIRST_CHARACTER,
+    INVALID_ESCAPE_SEQUENCE,
+  };
+
+  class parse_exception : public std::runtime_error {
+    using std::runtime_error::runtime_error;
+  };
+
+  json_pointer() = default;
+  ~json_pointer() = default;
+
+  /*
+   * Parse string into vector of unescaped tokens.
+   * Non-throwing and throwing versions.
+   */
+  static Expected<json_pointer, parse_error> try_parse(StringPiece const str);
+
+  static json_pointer parse(StringPiece const str);
+
+  /*
+   * Get access to the parsed tokens for applications that want to traverse
+   * the pointer.
+   */
+  std::vector<std::string> const& tokens() const;
+
+ private:
+  explicit json_pointer(std::vector<std::string>) noexcept;
+
+  /*
+   * Unescape the specified escape sequences, returns false if incorrect
+   */
+  static bool unescape(std::string&);
+
+  std::vector<std::string> tokens_;
+};
+
+} // namespace folly

folly/test/DynamicTest.cpp

@@ -756,3 +756,68 @@ TEST(Dynamic, MergeDiffNestedObjects) {
   source.merge_patch(patch);
   EXPECT_EQ(source, target);
 }
+
+using folly::json_pointer;
+
+TEST(Dynamic, JSONPointer) {
+  dynamic target = dynamic::object;
+  dynamic ary = dynamic::array("bar", "baz", dynamic::array("bletch", "xyzzy"));
+  target["foo"] = ary;
+  target[""] = 0;
+  target["a/b"] = 1;
+  target["c%d"] = 2;
+  target["e^f"] = 3;
+  target["g|h"] = 4;
+  target["i\\j"] = 5;
+  target["k\"l"] = 6;
+  target[" "] = 7;
+  target["m~n"] = 8;
+  target["xyz"] = dynamic::object;
+  target["xyz"][""] = dynamic::object("nested", "abc");
+  target["xyz"]["def"] = dynamic::array(1, 2, 3);
+  target["long_array"] = dynamic::array(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);
+  target["-"] = dynamic::object("x", "y");
+
+  EXPECT_EQ(target, *target.get_ptr(json_pointer::parse("")));
+  EXPECT_EQ(ary, *(target.get_ptr(json_pointer::parse("/foo"))));
+  EXPECT_EQ("bar", target.get_ptr(json_pointer::parse("/foo/0"))->getString());
+  EXPECT_EQ(0, target.get_ptr(json_pointer::parse("/"))->getInt());
+  EXPECT_EQ(1, target.get_ptr(json_pointer::parse("/a~1b"))->getInt());
+  EXPECT_EQ(2, target.get_ptr(json_pointer::parse("/c%d"))->getInt());
+  EXPECT_EQ(3, target.get_ptr(json_pointer::parse("/e^f"))->getInt());
+  EXPECT_EQ(4, target.get_ptr(json_pointer::parse("/g|h"))->getInt());
+  EXPECT_EQ(5, target.get_ptr(json_pointer::parse("/i\\j"))->getInt());
+  EXPECT_EQ(6, target.get_ptr(json_pointer::parse("/k\"l"))->getInt());
+  EXPECT_EQ(7, target.get_ptr(json_pointer::parse("/ "))->getInt());
+  EXPECT_EQ(8, target.get_ptr(json_pointer::parse("/m~0n"))->getInt());
+  // empty key in path
+  EXPECT_EQ(
+      "abc", target.get_ptr(json_pointer::parse("/xyz//nested"))->getString());
+  EXPECT_EQ(3, target.get_ptr(json_pointer::parse("/xyz/def/2"))->getInt());
+  EXPECT_EQ("baz", ary.get_ptr(json_pointer::parse("/1"))->getString());
+  EXPECT_EQ("bletch", ary.get_ptr(json_pointer::parse("/2/0"))->getString());
+  // double-digit index
+  EXPECT_EQ(
+      12, target.get_ptr(json_pointer::parse("/long_array/11"))->getInt());
+  // allow '-' to index in objects
+  EXPECT_EQ("y", target.get_ptr(json_pointer::parse("/-/x"))->getString());
+
+  // invalid JSON pointers formatting when accessing array
+  EXPECT_THROW(
+      target.get_ptr(json_pointer::parse("/foo/01")), std::invalid_argument);
+
+  // non-existent keys/indexes
+  EXPECT_EQ(nullptr, ary.get_ptr(json_pointer::parse("/3")));
+  EXPECT_EQ(nullptr, target.get_ptr(json_pointer::parse("/unknown_key")));
+  // intermediate key not found
+  EXPECT_EQ(nullptr, target.get_ptr(json_pointer::parse("/foox/test")));
+  // Intermediate key is '-'
+  EXPECT_EQ(nullptr, target.get_ptr(json_pointer::parse("/foo/-/key")));
+
+  // invalid path in object (key in array)
+  EXPECT_THROW(
+      target.get_ptr(json_pointer::parse("/foo/1/bar")), folly::TypeError);
+
+  // Allow "-" index in the array
+  EXPECT_EQ(nullptr, target.get_ptr(json_pointer::parse("/foo/-")));
+}

folly/test/Makefile.am

@@ -133,6 +133,10 @@ json_test_SOURCES = JsonTest.cpp
 json_test_LDADD = libfollytestmain.la $(top_builddir)/libfollybenchmark.la
 TESTS += json_test
 
+json_pointer_test_SOURCES = json_pointer_test.cpp
+json_pointer_test_LDADD = libfollytestmain.la
+TESTS += json_pointer_test
+
 benchmark_test_SOURCES = BenchmarkTest.cpp
 benchmark_test_LDADD = libfollytestmain.la $(top_builddir)/libfollybenchmark.la
 check_PROGRAMS += benchmark_test

folly/test/json_pointer_test.cpp

+/*
+ * Copyright 2011-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/json_pointer.h>
+#include <folly/portability/GMock.h>
+#include <folly/portability/GTest.h>
+
+using folly::json_pointer;
+using ::testing::ElementsAreArray;
+
+class JsonPointerTest : public ::testing::Test {};
+
+TEST_F(JsonPointerTest, ValidPointers) {
+  EXPECT_THAT(
+      json_pointer::parse("").tokens(),
+      ElementsAreArray(std::vector<std::string>{}));
+  EXPECT_THAT(json_pointer::parse("/").tokens(), ElementsAreArray({""}));
+  EXPECT_THAT(
+      json_pointer::parse("/1/2/3").tokens(),
+      ElementsAreArray({"1", "2", "3"}));
+  EXPECT_THAT(
+      json_pointer::parse("/~0~1/~0/10").tokens(),
+      ElementsAreArray({"~/", "~", "10"}));
+}
+
+TEST_F(JsonPointerTest, InvalidPointers) {
+  EXPECT_EQ(
+      json_pointer::parse_error::INVALID_FIRST_CHARACTER,
+      json_pointer::try_parse("a").error());
+  EXPECT_EQ(
+      json_pointer::parse_error::INVALID_ESCAPE_SEQUENCE,
+      json_pointer::try_parse("/~").error());
+  EXPECT_EQ(
+      json_pointer::parse_error::INVALID_ESCAPE_SEQUENCE,
+      json_pointer::try_parse("/~x").error());
+}