Merge pull request #1662 from OmnipotentEntity/develop

Add binary type support to all binary file formats, as well as an internally represented binary type

include/nlohmann/detail/conversions/to_json.hpp

@@ -67,6 +67,28 @@ struct external_constructor<value_t::string>
     }
 };
 
+template<>
+struct external_constructor<value_t::binary>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b)
+    {
+        j.m_type = value_t::binary;
+        typename BasicJsonType::internal_binary_t value{b};
+        j.m_value = value;
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b)
+    {
+        j.m_type = value_t::binary;
+        typename BasicJsonType::internal_binary_t value{std::move(b)};
+        j.m_value = value;
+        j.assert_invariant();
+    }
+};
+
 template<>
 struct external_constructor<value_t::number_float>
 {

include/nlohmann/detail/input/json_sax.hpp

@@ -31,6 +31,7 @@ struct json_sax
     using number_float_t = typename BasicJsonType::number_float_t;
     /// type for strings
     using string_t = typename BasicJsonType::string_t;
+    using binary_t = typename BasicJsonType::binary_t;
 
     /*!
     @brief a null value was read
@@ -75,6 +76,14 @@ struct json_sax
     */
     virtual bool string(string_t& val) = 0;
 
+    /*!
+    @brief a binary string was read
+    @param[in] val  binary value
+    @return whether parsing should proceed
+    @note It is safe to move the passed binary.
+    */
+    virtual bool binary(binary_t& val) = 0;
+
     /*!
     @brief the beginning of an object was read
     @param[in] elements  number of object elements or -1 if unknown
@@ -149,6 +158,7 @@ class json_sax_dom_parser
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
     using number_float_t = typename BasicJsonType::number_float_t;
     using string_t = typename BasicJsonType::string_t;
+    using binary_t = typename BasicJsonType::binary_t;
 
     /*!
     @param[in, out] r  reference to a JSON value that is manipulated while
@@ -202,6 +212,12 @@ class json_sax_dom_parser
         return true;
     }
 
+    bool binary(binary_t& val)
+    {
+        handle_binary(val);
+        return true;
+    }
+
     bool start_object(std::size_t len)
     {
         ref_stack.push_back(handle_value(BasicJsonType::value_t::object));
@@ -311,6 +327,36 @@ class json_sax_dom_parser
         return object_element;
     }
 
+    /*!
+    @invariant If the ref stack is empty, then the passed value will be the new
+               root.
+    @invariant If the ref stack contains a value, then it is an array or an
+               object to which we can add elements
+    */
+    template<typename BinaryValue>
+    JSON_HEDLEY_RETURNS_NON_NULL
+    BasicJsonType* handle_binary(BinaryValue&& v)
+    {
+        if (ref_stack.empty())
+        {
+            root = BasicJsonType::binary_array(std::forward<BinaryValue>(v));
+            return &root;
+        }
+
+        assert(ref_stack.back()->is_array() or ref_stack.back()->is_object());
+
+        if (ref_stack.back()->is_array())
+        {
+            ref_stack.back()->m_value.array->emplace_back(BasicJsonType::binary_array(std::forward<BinaryValue>(v)));
+            return &(ref_stack.back()->m_value.array->back());
+        }
+
+        assert(ref_stack.back()->is_object());
+        assert(object_element);
+        *object_element = BasicJsonType::binary_array(std::forward<BinaryValue>(v));
+        return object_element;
+    }
+
     /// the parsed JSON value
     BasicJsonType& root;
     /// stack to model hierarchy of values
@@ -331,6 +377,7 @@ class json_sax_dom_callback_parser
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
     using number_float_t = typename BasicJsonType::number_float_t;
     using string_t = typename BasicJsonType::string_t;
+    using binary_t = typename BasicJsonType::binary_t;
     using parser_callback_t = typename BasicJsonType::parser_callback_t;
     using parse_event_t = typename BasicJsonType::parse_event_t;
 
@@ -385,6 +432,12 @@ class json_sax_dom_callback_parser
         return true;
     }
 
+    bool binary(binary_t& val)
+    {
+        handle_value(val);
+        return true;
+    }
+
     bool start_object(std::size_t len)
     {
         // check callback for object start
@@ -635,6 +688,7 @@ class json_sax_acceptor
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
     using number_float_t = typename BasicJsonType::number_float_t;
     using string_t = typename BasicJsonType::string_t;
+    using binary_t = typename BasicJsonType::binary_t;
 
     bool null()
     {
@@ -666,7 +720,12 @@ class json_sax_acceptor
         return true;
     }
 
-    bool start_object(std::size_t  /*unused*/ = std::size_t(-1))
+    bool binary(binary_t& /*unused*/)
+    {
+        return true;
+    }
+
+    bool start_object(std::size_t /*unused*/ = std::size_t(-1))
     {
         return true;
     }
@@ -681,7 +740,7 @@ class json_sax_acceptor
         return true;
     }
 
-    bool start_array(std::size_t  /*unused*/ = std::size_t(-1))
+    bool start_array(std::size_t /*unused*/ = std::size_t(-1))
     {
         return true;
     }

include/nlohmann/detail/iterators/internal_iterator.hpp

@@ -18,6 +18,8 @@ template<typename BasicJsonType> struct internal_iterator
     typename BasicJsonType::object_t::iterator object_iterator {};
     /// iterator for JSON arrays
     typename BasicJsonType::array_t::iterator array_iterator {};
+    /// iterator for JSON binary arrays
+    typename BasicJsonType::binary_t::iterator binary_iterator {};
     /// generic iterator for all other types
     primitive_iterator_t primitive_iterator {};
 };

include/nlohmann/detail/macro_scope.hpp

@@ -113,9 +113,10 @@
              class StringType, class BooleanType, class NumberIntegerType, \
              class NumberUnsignedType, class NumberFloatType,              \
              template<typename> class AllocatorType,                       \
-             template<typename, typename = void> class JSONSerializer>
+             template<typename, typename = void> class JSONSerializer,     \
+             class BinaryType>
 
 #define NLOHMANN_BASIC_JSON_TPL                                            \
     basic_json<ObjectType, ArrayType, StringType, BooleanType,             \
     NumberIntegerType, NumberUnsignedType, NumberFloatType,                \
-    AllocatorType, JSONSerializer>
+    AllocatorType, JSONSerializer, BinaryType>

include/nlohmann/detail/output/binary_writer.hpp

@@ -26,6 +26,7 @@ template<typename BasicJsonType, typename CharType>
 class binary_writer
 {
     using string_t = typename BasicJsonType::string_t;
+    using internal_binary_t = typename BasicJsonType::internal_binary_t;
 
   public:
     /*!
@@ -258,6 +259,45 @@ class binary_writer
                 break;
             }
 
+            case value_t::binary:
+            {
+                // step 1: write control byte and the binary array size
+                const auto N = j.m_value.binary->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<std::uint8_t>(0x40 + N));
+                }
+                else if (N <= (std::numeric_limits<std::uint8_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x58));
+                    write_number(static_cast<std::uint8_t>(N));
+                }
+                else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x59));
+                    write_number(static_cast<std::uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x5A));
+                    write_number(static_cast<std::uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= (std::numeric_limits<std::uint64_t>::max)())
+                {
+                    oa->write_character(to_char_type(0x5B));
+                    write_number(static_cast<std::uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write each element
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.binary->data()),
+                    N);
+
+                break;
+            }
+
             case value_t::object:
             {
                 // step 1: write control byte and the object size
@@ -506,6 +546,101 @@ class binary_writer
                 break;
             }
 
+            case value_t::binary:
+            {
+                // step 0: determine if the binary type has a set subtype to
+                // determine whether or not to use the ext or fixext types
+                const bool use_ext = j.m_value.binary->has_subtype;
+
+                // step 1: write control byte and the byte string length
+                const auto N = j.m_value.binary->size();
+                if (N <= (std::numeric_limits<std::uint8_t>::max)())
+                {
+                    std::uint8_t output_type;
+                    bool fixed = true;
+                    if (use_ext)
+                    {
+                        switch (N)
+                        {
+                            case 1:
+                                output_type = 0xD4; // fixext 1
+                                break;
+                            case 2:
+                                output_type = 0xD5; // fixext 2
+                                break;
+                            case 4:
+                                output_type = 0xD6; // fixext 4
+                                break;
+                            case 8:
+                                output_type = 0xD7; // fixext 8
+                                break;
+                            case 16:
+                                output_type = 0xD8; // fixext 16
+                                break;
+                            default:
+                                output_type = 0xC7; // ext 8
+                                fixed = false;
+                                break;
+                        }
+
+                    }
+                    else
+                    {
+                        output_type = 0xC4; // bin 8
+                        fixed = false;
+                    }
+
+                    oa->write_character(to_char_type(output_type));
+                    if (not fixed)
+                    {
+                        write_number(static_cast<std::uint8_t>(N));
+                    }
+                }
+                else if (N <= (std::numeric_limits<std::uint16_t>::max)())
+                {
+                    std::uint8_t output_type;
+                    if (use_ext)
+                    {
+                        output_type = 0xC8; // ext 16
+                    }
+                    else
+                    {
+                        output_type = 0xC5; // bin 16
+                    }
+
+                    oa->write_character(to_char_type(output_type));
+                    write_number(static_cast<std::uint16_t>(N));
+                }
+                else if (N <= (std::numeric_limits<std::uint32_t>::max)())
+                {
+                    std::uint8_t output_type;
+                    if (use_ext)
+                    {
+                        output_type = 0xC9; // ext 32
+                    }
+                    else
+                    {
+                        output_type = 0xC6; // bin 32
+                    }
+
+                    oa->write_character(to_char_type(output_type));
+                    write_number(static_cast<std::uint32_t>(N));
+                }
+
+                // step 1.5: if this is an ext type, write the subtype
+                if (use_ext)
+                {
+                    write_number(j.m_value.binary->subtype);
+                }
+
+                // step 2: write the byte string
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.binary->data()),
+                    N);
+
+                break;
+            }
+
             case value_t::object:
             {
                 // step 1: write control byte and the object size
@@ -649,6 +784,49 @@ class binary_writer
                 break;
             }
 
+            case value_t::binary:
+            {
+                if (add_prefix)
+                {
+                    oa->write_character(to_char_type('['));
+                }
+
+                if (use_type and not j.m_value.binary->empty())
+                {
+                    assert(use_count);
+                    oa->write_character(to_char_type('$'));
+                    oa->write_character('U');
+                }
+
+                if (use_count)
+                {
+                    oa->write_character(to_char_type('#'));
+                    write_number_with_ubjson_prefix(j.m_value.binary->size(), true);
+                }
+
+                if (use_type)
+                {
+                    oa->write_characters(
+                        reinterpret_cast<const CharType*>(j.m_value.binary->data()),
+                        j.m_value.binary->size());
+                }
+                else
+                {
+                    for (size_t i = 0; i < j.m_value.binary->size(); ++i)
+                    {
+                        oa->write_character(to_char_type('U'));
+                        oa->write_character(j.m_value.binary->data()[i]);
+                    }
+                }
+
+                if (not use_count)
+                {
+                    oa->write_character(to_char_type(']'));
+                }
+
+                break;
+            }
+
             case value_t::object:
             {
                 if (add_prefix)
@@ -871,6 +1049,14 @@ class binary_writer
         return sizeof(std::int32_t) + embedded_document_size + 1ul;
     }
 
+    /*!
+    @return The size of the BSON-encoded binary array @a value
+    */
+    static std::size_t calc_bson_binary_size(const typename BasicJsonType::internal_binary_t& value)
+    {
+        return sizeof(std::int32_t) + value.size() + 1ul;
+    }
+
     /*!
     @brief Writes a BSON element with key @a name and array @a value
     */
@@ -890,6 +1076,27 @@ class binary_writer
         oa->write_character(to_char_type(0x00));
     }
 
+    /*!
+    @brief Writes a BSON element with key @a name and binary value @a value
+    */
+    void write_bson_binary(const string_t& name,
+                           const internal_binary_t& value)
+    {
+        write_bson_entry_header(name, 0x05);
+
+        write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size()));
+        std::uint8_t subtype = 0x00; // Generic Binary Subtype
+        if (value.has_subtype)
+        {
+            subtype = value.subtype;
+        }
+        write_number(subtype);
+
+        oa->write_characters(
+            reinterpret_cast<const CharType*>(value.data()),
+            value.size());
+    }
+
     /*!
     @brief Calculates the size necessary to serialize the JSON value @a j with its @a name
     @return The calculated size for the BSON document entry for @a j with the given @a name.
@@ -906,6 +1113,9 @@ class binary_writer
             case value_t::array:
                 return header_size + calc_bson_array_size(*j.m_value.array);
 
+            case value_t::binary:
+                return header_size + calc_bson_binary_size(*j.m_value.binary);
+
             case value_t::boolean:
                 return header_size + 1ul;
 
@@ -950,6 +1160,9 @@ class binary_writer
             case value_t::array:
                 return write_bson_array(name, *j.m_value.array);
 
+            case value_t::binary:
+                return write_bson_binary(name, *j.m_value.binary);
+
             case value_t::boolean:
                 return write_bson_boolean(name, j.m_value.boolean);
 
@@ -1230,7 +1443,8 @@ class binary_writer
             case value_t::string:
                 return 'S';
 
-            case value_t::array:
+            case value_t::array: // fallthrough
+            case value_t::binary:
                 return '[';
 
             case value_t::object:

include/nlohmann/detail/output/serializer.hpp

@@ -45,6 +45,7 @@ class serializer
     using number_float_t = typename BasicJsonType::number_float_t;
     using number_integer_t = typename BasicJsonType::number_integer_t;
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using binary_t = typename BasicJsonType::binary_t;
     static constexpr std::uint8_t UTF8_ACCEPT = 0;
     static constexpr std::uint8_t UTF8_REJECT = 1;
 
@@ -83,16 +84,19 @@ class serializer
     - strings and object keys are escaped using `escape_string()`
     - integer numbers are converted implicitly via `operator<<`
     - floating-point numbers are converted to a string using `"%g"` format
+    - if specified to, binary values are output using the syntax `b[]`, otherwise an exception is thrown
 
-    @param[in] val             value to serialize
-    @param[in] pretty_print    whether the output shall be pretty-printed
-    @param[in] indent_step     the indent level
-    @param[in] current_indent  the current indent level (only used internally)
+    @param[in] val               value to serialize
+    @param[in] pretty_print      whether the output shall be pretty-printed
+    @param[in] indent_step       the indent level
+    @param[in] current_indent    the current indent level (only used internally)
+    @param[in] serialize_binary  whether the output shall include non-standard binary output
     */
     void dump(const BasicJsonType& val, const bool pretty_print,
               const bool ensure_ascii,
               const unsigned int indent_step,
-              const unsigned int current_indent = 0)
+              const unsigned int current_indent = 0,
+              const bool serialize_binary = false)
     {
         switch (val.m_type)
         {
@@ -236,6 +240,55 @@ class serializer
                 return;
             }
 
+            case value_t::binary:
+            {
+                if (not serialize_binary)
+                {
+                    JSON_THROW(type_error::create(317, "cannot serialize binary data to text JSON"));
+                }
+
+                if (val.m_value.binary->empty())
+                {
+                    o->write_characters("b[]", 3);
+                }
+                else if (pretty_print)
+                {
+                    o->write_characters("b[", 2);
+                    for (auto i = val.m_value.binary->cbegin();
+                            i != val.m_value.binary->cend() - 1; ++i)
+                    {
+                        dump_integer(*i);
+                        o->write_character(',');
+                        int index = i - val.m_value.binary->cbegin();
+                        if (index % 16 == 0)
+                        {
+                            o->write_character('\n');
+                        }
+                        else
+                        {
+                            o->write_character(' ');
+                        }
+                    }
+
+                    dump_integer(val.m_value.binary->back());
+                    o->write_character(']');
+                }
+                else
+                {
+                    o->write_characters("b[", 2);
+                    for (auto i = val.m_value.binary->cbegin();
+                            i != val.m_value.binary->cend() - 1; ++i)
+                    {
+                        dump_integer(*i);
+                        o->write_character(',');
+                    }
+
+                    dump_integer(val.m_value.binary->back());
+                    o->write_character(']');
+                }
+                return;
+            }
+
             case value_t::boolean:
             {
                 if (val.m_value.boolean)
@@ -592,7 +645,8 @@ class serializer
     */
     template<typename NumberType, detail::enable_if_t<
                  std::is_same<NumberType, number_unsigned_t>::value or
-                 std::is_same<NumberType, number_integer_t>::value,
+                 std::is_same<NumberType, number_integer_t>::value or
+                 std::is_same<NumberType, typename binary_t::value_type>::value,
                  int> = 0>
     void dump_integer(NumberType x)
     {
@@ -630,7 +684,7 @@ class serializer
         if (is_negative)
         {
             *buffer_ptr = '-';
-            abs_value = remove_sign(x);
+            abs_value = remove_sign(static_cast<number_integer_t>(x));
 
             // account one more byte for the minus sign
             n_chars = 1 + count_digits(abs_value);

include/nlohmann/detail/value_t.hpp

@@ -48,6 +48,7 @@ enum class value_t : std::uint8_t
     number_integer,   ///< number value (signed integer)
     number_unsigned,  ///< number value (unsigned integer)
     number_float,     ///< number value (floating-point)
+    binary,           ///< binary array (ordered collection of bytes)
     discarded         ///< discarded by the parser callback function
 };
 
@@ -55,17 +56,21 @@ enum class value_t : std::uint8_t
 @brief comparison operator for JSON types
 
 Returns an ordering that is similar to Python:
-- order: null < boolean < number < object < array < string
+- order: null < boolean < number < object < array < string < binary
 - furthermore, each type is not smaller than itself
 - discarded values are not comparable
+- binary is represented as a b"" string in python and directly comparable to a
+  string; however, making a binary array directly comparable with a string would
+  be surprising behavior in a JSON file.
 
 @since version 1.0.0
 */
 inline bool operator<(const value_t lhs, const value_t rhs) noexcept
 {
-    static constexpr std::array<std::uint8_t, 8> order = {{
+    static constexpr std::array<std::uint8_t, 9> order = {{
             0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */,
-            1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */
+            1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */,
+            6 /* binary */
         }
     };
 

include/nlohmann/json_fwd.hpp

@@ -33,7 +33,8 @@ template<template<typename U, typename V, typename... Args> class ObjectType =
          class NumberFloatType = double,
          template<typename U> class AllocatorType = std::allocator,
          template<typename T, typename SFINAE = void> class JSONSerializer =
-         adl_serializer>
+         adl_serializer,
+         class BinaryType = std::vector<std::uint8_t>>
 class basic_json;
 
 /*!

test/data/binary_data/cbor_binary.out

+








	
	
+
	
+
	
+
	
+

	
+

	
+

	
+

	
+

	
+

	
+

	
+

	
+

	
+

	
+
KxY?3"xUF*RPg)KxY?3"xUF*RPg)KxY?3"xUF*RPg)KxY?3"xUF*RPg)
	
+
	
+
\ No newline at end of file

test/src/unit-bson.cpp

@@ -492,6 +492,36 @@ TEST_CASE("BSON")
             CHECK(json::from_bson(result, true, false) == j);
         }
 
+        SECTION("non-empty object with binary member")
+        {
+            const size_t N = 10;
+            const auto s = std::vector<uint8_t>(N, 'x');
+            json j =
+            {
+                { "entry", json::binary_array(s) }
+            };
+
+            std::vector<uint8_t> expected =
+            {
+                0x1B, 0x00, 0x00, 0x00, // size (little endian)
+                0x05, // entry: binary
+                'e', 'n', 't', 'r', 'y', '\x00',
+
+                0x0A, 0x00, 0x00, 0x00, // size of binary (little endian)
+                0x00, // Generic binary subtype
+                0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78, 0x78,
+
+                0x00 // end marker
+            };
+
+            const auto result = json::to_bson(j);
+            CHECK(result == expected);
+
+            // roundtrip
+            CHECK(json::from_bson(result) == j);
+            CHECK(json::from_bson(result, true, false) == j);
+        }
+
         SECTION("Some more complex document")
         {
             // directly encoding uint64 is not supported in bson (only for timestamp values)
@@ -646,6 +676,11 @@ class SaxCountdown
         return events_left-- > 0;
     }
 
+    bool binary(std::vector<uint8_t>&)
+    {
+        return events_left-- > 0;
+    }
+
     bool start_object(std::size_t)
     {
         return events_left-- > 0;

test/src/unit-cbor.cpp

@@ -36,6 +36,7 @@ using nlohmann::json;
 #include <fstream>
 #include <sstream>
 #include <iomanip>
+#include <iostream>
 #include <set>
 
 namespace
@@ -76,6 +77,11 @@ class SaxCountdown
         return events_left-- > 0;
     }
 
+    bool binary(std::vector<std::uint8_t>&)
+    {
+        return events_left-- > 0;
+    }
+
     bool start_object(std::size_t)
     {
         return events_left-- > 0;
@@ -1285,10 +1291,156 @@ TEST_CASE("CBOR")
                 CHECK(json::from_cbor(result, true, false) == j);
             }
         }
+
+        SECTION("binary")
+        {
+            SECTION("N = 0..23")
+            {
+                for (size_t N = 0; N <= 0x17; ++N)
+                {
+                    CAPTURE(N)
+
+                    // create JSON value with byte array containing of N * 'x'
+                    const auto s = std::vector<uint8_t>(N, 'x');
+                    json j = json::binary_array(s);
+
+                    // create expected byte vector
+                    std::vector<uint8_t> expected;
+                    expected.push_back(static_cast<uint8_t>(0x40 + N));
+                    for (size_t i = 0; i < N; ++i)
+                    {
+                        expected.push_back(0x78);
+                    }
+
+                    // compare result + size
+                    const auto result = json::to_cbor(j);
+                    CHECK(result == expected);
+                    CHECK(result.size() == N + 1);
+                    // check that no null byte is appended
+                    if (N > 0)
+                    {
+                        CHECK(result.back() != '\x00');
+                    }
+
+                    // roundtrip
+                    CHECK(json::from_cbor(result) == j);
+                    CHECK(json::from_cbor(result, true, false) == j);
+                }
+            }
+
+            SECTION("N = 24..255")
+            {
+                for (size_t N = 24; N <= 255; ++N)
+                {
+                    CAPTURE(N)
+
+                    // create JSON value with string containing of N * 'x'
+                    const auto s = std::vector<uint8_t>(N, 'x');
+                    json j = json::binary_array(s);
+
+                    // create expected byte vector
+                    std::vector<uint8_t> expected;
+                    expected.push_back(0x58);
+                    expected.push_back(static_cast<uint8_t>(N));
+                    for (size_t i = 0; i < N; ++i)
+                    {
+                        expected.push_back('x');
+                    }
+
+                    // compare result + size
+                    const auto result = json::to_cbor(j);
+                    CHECK(result == expected);
+                    CHECK(result.size() == N + 2);
+                    // check that no null byte is appended
+                    CHECK(result.back() != '\x00');
+
+                    // roundtrip
+                    CHECK(json::from_cbor(result) == j);
+                    CHECK(json::from_cbor(result, true, false) == j);
+                }
+            }
+
+            SECTION("N = 256..65535")
+            {
+                for (size_t N :
+                        {
+                            256u, 999u, 1025u, 3333u, 2048u, 65535u
+                        })
+                {
+                    CAPTURE(N)
+
+                    // create JSON value with string containing of N * 'x'
+                    const auto s = std::vector<uint8_t>(N, 'x');
+                    json j = json::binary_array(s);
+
+                    // create expected byte vector (hack: create string first)
+                    std::vector<uint8_t> expected(N, 'x');
+                    // reverse order of commands, because we insert at begin()
+                    expected.insert(expected.begin(), static_cast<uint8_t>(N & 0xff));
+                    expected.insert(expected.begin(), static_cast<uint8_t>((N >> 8) & 0xff));
+                    expected.insert(expected.begin(), 0x59);
+
+                    // compare result + size
+                    const auto result = json::to_cbor(j);
+                    CHECK(result == expected);
+                    CHECK(result.size() == N + 3);
+                    // check that no null byte is appended
+                    CHECK(result.back() != '\x00');
+
+                    // roundtrip
+                    CHECK(json::from_cbor(result) == j);
+                    CHECK(json::from_cbor(result, true, false) == j);
+                }
+            }
+
+            SECTION("N = 65536..4294967295")
+            {
+                for (size_t N :
+                        {
+                            65536u, 77777u, 1048576u
+                        })
+                {
+                    CAPTURE(N)
+
+                    // create JSON value with string containing of N * 'x'
+                    const auto s = std::vector<uint8_t>(N, 'x');
+                    json j = json::binary_array(s);
+
+                    // create expected byte vector (hack: create string first)
+                    std::vector<uint8_t> expected(N, 'x');
+                    // reverse order of commands, because we insert at begin()
+                    expected.insert(expected.begin(), static_cast<uint8_t>(N & 0xff));
+                    expected.insert(expected.begin(), static_cast<uint8_t>((N >> 8) & 0xff));
+                    expected.insert(expected.begin(), static_cast<uint8_t>((N >> 16) & 0xff));
+                    expected.insert(expected.begin(), static_cast<uint8_t>((N >> 24) & 0xff));
+                    expected.insert(expected.begin(), 0x5a);
+
+                    // compare result + size
+                    const auto result = json::to_cbor(j);
+                    CHECK(result == expected);
+                    CHECK(result.size() == N + 5);
+                    // check that no null byte is appended
+                    CHECK(result.back() != '\x00');
+
+                    // roundtrip
+                    CHECK(json::from_cbor(result) == j);
+                    CHECK(json::from_cbor(result, true, false) == j);
+                }
+            }
+        }
     }
 
     SECTION("additional deserialization")
     {
+        SECTION("0x5b (byte array)")
+        {
+            std::vector<uint8_t> given = {0x5b, 0x00, 0x00, 0x00, 0x00,
+                                          0x00, 0x00, 0x00, 0x01, 0x61
+                                         };
+            json j = json::from_cbor(given);
+            CHECK(j == json::binary_array(std::vector<uint8_t> {'a'}));
+        }
+
         SECTION("0x7b (string)")
         {
             std::vector<uint8_t> given = {0x7b, 0x00, 0x00, 0x00, 0x00,
@@ -1455,14 +1607,8 @@ TEST_CASE("CBOR")
                             0x1c, 0x1d, 0x1e, 0x1f,
                             // ?
                             0x3c, 0x3d, 0x3e, 0x3f,
-                            // byte strings
-                            0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
-                            // byte strings
-                            0x58, 0x59, 0x5a, 0x5b,
                             // ?
                             0x5c, 0x5d, 0x5e,
-                            // byte string
-                            0x5f,
                             // ?
                             0x7c, 0x7d, 0x7e,
                             // ?
@@ -1929,12 +2075,6 @@ TEST_CASE("all CBOR first bytes")
     {
         //// types not supported by this library
 
-        // byte strings
-        0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
-        0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
-        0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
-        // byte strings
-        0x58, 0x59, 0x5a, 0x5b, 0x5f,
         // date/time
         0xc0, 0xc1,
         // bignum
@@ -2144,6 +2284,20 @@ TEST_CASE("examples from RFC 7049 Appendix A")
         CHECK(json::parse("\"streaming\"") == json::from_cbor(std::vector<uint8_t>({0x7f, 0x65, 0x73, 0x74, 0x72, 0x65, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x67, 0xff})));
     }
 
+    SECTION("byte arrays")
+    {
+        std::ifstream f_cbor("test/data/binary_data/cbor_binary.cbor", std::ios::binary);
+        std::vector<uint8_t> packed((std::istreambuf_iterator<char>(f_cbor)),
+                                    std::istreambuf_iterator<char>());
+        json j;
+        CHECK_NOTHROW(j = json::from_cbor(packed));
+
+        std::ifstream f_bin("test/data/binary_data/cbor_binary.out", std::ios::binary);
+        std::vector<uint8_t> expected((std::istreambuf_iterator<char>(f_bin)),
+                                      std::istreambuf_iterator<char>());
+        CHECK(j == json::binary_array(expected));
+    }
+
     SECTION("arrays")
     {
         CHECK(json::to_cbor(json::parse("[]")) == std::vector<uint8_t>({0x80}));

test/src/unit-class_parser.cpp

@@ -77,6 +77,21 @@ class SaxEventLogger
         return true;
     }
 
+    bool binary(std::vector<std::uint8_t>& val)
+    {
+        std::string binary_contents = "binary(";
+        std::string comma_space = "";
+        for (auto b : val)
+        {
+            binary_contents.append(comma_space);
+            binary_contents.append(std::to_string(static_cast<int>(b)));
+            comma_space = ", ";
+        }
+        binary_contents.append(")");
+        events.push_back(binary_contents);
+        return true;
+    }
+
     bool start_object(std::size_t elements)
     {
         if (elements == std::size_t(-1))
@@ -168,6 +183,11 @@ class SaxCountdown : public nlohmann::json::json_sax_t
         return events_left-- > 0;
     }
 
+    bool binary(std::vector<std::uint8_t>&) override
+    {
+        return events_left-- > 0;
+    }
+
     bool start_object(std::size_t) override
     {
         return events_left-- > 0;

test/src/unit-deserialization.cpp

@@ -76,6 +76,21 @@ struct SaxEventLogger : public nlohmann::json_sax<json>
         return true;
     }
 
+    bool binary(std::vector<std::uint8_t>& val)
+    {
+        std::string binary_contents = "binary(";
+        std::string comma_space = "";
+        for (auto b : val)
+        {
+            binary_contents.append(comma_space);
+            binary_contents.append(std::to_string(static_cast<int>(b)));
+            comma_space = ", ";
+        }
+        binary_contents.append(")");
+        events.push_back(binary_contents);
+        return true;
+    }
+
     bool start_object(std::size_t elements) override
     {
         if (elements == std::size_t(-1))

test/src/unit-regression.cpp

@@ -107,7 +107,7 @@ struct foo_serializer < T, typename std::enable_if < !std::is_same<foo, T>::valu
 }
 
 using foo_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t,
-      std::uint64_t, double, std::allocator, ns::foo_serializer>;
+      std::uint64_t, double, std::allocator, ns::foo_serializer, std::vector<std::uint8_t>>;
 
 /////////////////////////////////////////////////////////////////////
 // for #805

test/src/unit-ubjson.cpp

@@ -73,6 +73,11 @@ class SaxCountdown
         return events_left-- > 0;
     }
 
+    bool binary(std::vector<std::uint8_t>&)
+    {
+        return events_left-- > 0;
+    }
+
     bool start_object(std::size_t)
     {
         return events_left-- > 0;
@@ -905,6 +910,231 @@ TEST_CASE("UBJSON")
             }
         }
 
+        SECTION("binary")
+        {
+            SECTION("N = 0..127")
+            {
+                for (std::size_t N = 0; N <= 127; ++N)
+                {
+                    CAPTURE(N)
+
+                    // create JSON value with byte array containing of N * 'x'
+                    const auto s = std::vector<std::uint8_t>(N, 'x');
+                    json j = json::binary_array(s);
+
+                    // create expected byte vector
+                    std::vector<std::uint8_t> expected;
+                    expected.push_back(static_cast<std::uint8_t>('['));
+                    if (N != 0)
+                    {
+                        expected.push_back(static_cast<std::uint8_t>('$'));
+                        expected.push_back(static_cast<std::uint8_t>('U'));
+                    }
+                    expected.push_back(static_cast<std::uint8_t>('#'));
+                    expected.push_back(static_cast<std::uint8_t>('i'));
+                    expected.push_back(static_cast<std::uint8_t>(N));
+                    for (size_t i = 0; i < N; ++i)
+                    {
+                        expected.push_back(0x78);
+                    }
+
+                    // compare result + size
+                    const auto result = json::to_ubjson(j, true, true);
+                    CHECK(result == expected);
+                    if (N == 0)
+                    {
+                        CHECK(result.size() == N + 4);
+                    }
+                    else
+                    {
+                        CHECK(result.size() == N + 6);
+                    }
+
+                    // check that no null byte is appended
+                    if (N > 0)
+                    {
+                        CHECK(result.back() != '\x00');
+                    }
+
+                    // roundtrip only works to an array of numbers
+                    json j_out = s;
+                    CHECK(json::from_ubjson(result) == j_out);
+                    CHECK(json::from_ubjson(result, true, false) == j_out);
+                }
+            }
+
+            SECTION("N = 128..255")
+            {
+                for (std::size_t N = 128; N <= 255; ++N)
+                {
+                    CAPTURE(N)
+
+                    // create JSON value with byte array containing of N * 'x'
+                    const auto s = std::vector<std::uint8_t>(N, 'x');
+                    json j = json::binary_array(s);
+
+                    // create expected byte vector
+                    std::vector<uint8_t> expected;
+                    expected.push_back(static_cast<std::uint8_t>('['));
+                    expected.push_back(static_cast<std::uint8_t>('$'));
+                    expected.push_back(static_cast<std::uint8_t>('U'));
+                    expected.push_back(static_cast<std::uint8_t>('#'));
+                    expected.push_back(static_cast<std::uint8_t>('U'));
+                    expected.push_back(static_cast<std::uint8_t>(N));
+                    for (size_t i = 0; i < N; ++i)
+                    {
+                        expected.push_back(0x78);
+                    }
+
+                    // compare result + size
+                    const auto result = json::to_ubjson(j, true, true);
+                    CHECK(result == expected);
+                    CHECK(result.size() == N + 6);
+                    // check that no null byte is appended
+                    CHECK(result.back() != '\x00');
+
+                    // roundtrip only works to an array of numbers
+                    json j_out = s;
+                    CHECK(json::from_ubjson(result) == j_out);
+                    CHECK(json::from_ubjson(result, true, false) == j_out);
+                }
+            }
+
+            SECTION("N = 256..32767")
+            {
+                for (std::size_t N :
+                        {
+                            256u, 999u, 1025u, 3333u, 2048u, 32767u
+                        })
+                {
+                    CAPTURE(N)
+
+                    // create JSON value with byte array containing of N * 'x'
+                    const auto s = std::vector<std::uint8_t>(N, 'x');
+                    json j = json::binary_array(s);
+
+                    // create expected byte vector
+                    std::vector<std::uint8_t> expected(N + 7, 'x');
+                    expected[0] = '[';
+                    expected[1] = '$';
+                    expected[2] = 'U';
+                    expected[3] = '#';
+                    expected[4] = 'I';
+                    expected[5] = static_cast<std::uint8_t>((N >> 8) & 0xFF);
+                    expected[6] = static_cast<std::uint8_t>(N & 0xFF);
+
+                    // compare result + size
+                    const auto result = json::to_ubjson(j, true, true);
+                    CHECK(result == expected);
+                    CHECK(result.size() == N + 7);
+                    // check that no null byte is appended
+                    CHECK(result.back() != '\x00');
+
+                    // roundtrip only works to an array of numbers
+                    json j_out = s;
+                    CHECK(json::from_ubjson(result) == j_out);
+                    CHECK(json::from_ubjson(result, true, false) == j_out);
+                }
+            }
+
+            SECTION("N = 32768..2147483647")
+            {
+                for (std::size_t N :
+                        {
+                            32768u, 77777u, 1048576u
+                        })
+                {
+                    CAPTURE(N)
+
+                    // create JSON value with byte array containing of N * 'x'
+                    const auto s = std::vector<std::uint8_t>(N, 'x');
+                    json j = json::binary_array(s);
+
+                    // create expected byte vector
+                    std::vector<std::uint8_t> expected(N + 9, 'x');
+                    expected[0] = '[';
+                    expected[1] = '$';
+                    expected[2] = 'U';
+                    expected[3] = '#';
+                    expected[4] = 'l';
+                    expected[5] = static_cast<std::uint8_t>((N >> 24) & 0xFF);
+                    expected[6] = static_cast<std::uint8_t>((N >> 16) & 0xFF);
+                    expected[7] = static_cast<std::uint8_t>((N >> 8) & 0xFF);
+                    expected[8] = static_cast<std::uint8_t>(N & 0xFF);
+
+                    // compare result + size
+                    const auto result = json::to_ubjson(j, true, true);
+                    CHECK(result == expected);
+                    CHECK(result.size() == N + 9);
+                    // check that no null byte is appended
+                    CHECK(result.back() != '\x00');
+
+                    // roundtrip only works to an array of numbers
+                    json j_out = s;
+                    CHECK(json::from_ubjson(result) == j_out);
+                    CHECK(json::from_ubjson(result, true, false) == j_out);
+                }
+            }
+
+            SECTION("Other Serializations")
+            {
+                const std::size_t N = 10;
+                const auto s = std::vector<std::uint8_t>(N, 'x');
+                json j = json::binary_array(s);
+
+                SECTION("No Count No Type")
+                {
+                    std::vector<uint8_t> expected;
+                    expected.push_back(static_cast<std::uint8_t>('['));
+                    for (std::size_t i = 0; i < N; ++i)
+                    {
+                        expected.push_back(static_cast<std::uint8_t>('U'));
+                        expected.push_back(static_cast<std::uint8_t>(0x78));
+                    }
+                    expected.push_back(static_cast<std::uint8_t>(']'));
+
+                    // compare result + size
+                    const auto result = json::to_ubjson(j, false, false);
+                    CHECK(result == expected);
+                    CHECK(result.size() == N + 12);
+                    // check that no null byte is appended
+                    CHECK(result.back() != '\x00');
+
+                    // roundtrip only works to an array of numbers
+                    json j_out = s;
+                    CHECK(json::from_ubjson(result) == j_out);
+                    CHECK(json::from_ubjson(result, true, false) == j_out);
+                }
+
+                SECTION("Yes Count No Type")
+                {
+                    std::vector<std::uint8_t> expected;
+                    expected.push_back(static_cast<std::uint8_t>('['));
+                    expected.push_back(static_cast<std::uint8_t>('#'));
+                    expected.push_back(static_cast<std::uint8_t>('i'));
+                    expected.push_back(static_cast<std::uint8_t>(N));
+
+                    for (size_t i = 0; i < N; ++i)
+                    {
+                        expected.push_back(static_cast<std::uint8_t>('U'));
+                        expected.push_back(static_cast<std::uint8_t>(0x78));
+                    }
+
+                    // compare result + size
+                    const auto result = json::to_ubjson(j, true, false);
+                    CHECK(result == expected);
+                    CHECK(result.size() == N + 14);
+                    // check that no null byte is appended
+                    CHECK(result.back() != '\x00');
+
+                    // roundtrip only works to an array of numbers
+                    json j_out = s;
+                    CHECK(json::from_ubjson(result) == j_out);
+                    CHECK(json::from_ubjson(result, true, false) == j_out);
+                }
+            }
+        }
+
         SECTION("array")
         {
             SECTION("empty")

test/src/unit-udt.cpp

@@ -653,8 +653,7 @@ TEST_CASE("custom serializer for pods" * doctest::test_suite("udt"))
 {
     using custom_json =
         nlohmann::basic_json<std::map, std::vector, std::string, bool,
-        std::int64_t, std::uint64_t, double, std::allocator,
-        pod_serializer>;
+        std::int64_t, std::uint64_t, double, std::allocator, pod_serializer>;
 
     auto p = udt::small_pod{42, '/', 42};
     custom_json j = p;