Skip to content

F
folly
Commit a1107271 authored by Joe Richey's avatar Joe Richey Committed by Sara Golemon
Browse files
Options

Actually denote when we have infinite generators 

Summary: Before we didn't do anything at all with the ::infinite value for our
generators, now all the sources operators and sinks are properly notated. The
one signifcant change regarding what is allowed concerns 'cycle' which now (when
called with no arguments) always produces an infinite generator. This is fine
for all but the weirdest of generators fed into cycle.

Reviewed By: @ddrcoder

Differential Revision: D2240328
parent 43d53e06
Hide whitespace changes
Inline Side-by-side
Showing with 1031 additions and 934 deletions
folly/gen/Base-inl.h
View file @ a1107271
...@@ -37,6 +37,9 @@ struct ArgumentReference ...@@ -37,6 +37,9 @@ struct ArgumentReference
T&& // int -> int&& T&& // int -> int&&
>::type> {}; >::type> {};
/**
* Group - The output objects from the GroupBy operator
*/
template <class Key, class Value> template <class Key, class Value>
class Group : public GenImpl<Value&&, Group<Key, Value>> { class Group : public GenImpl<Value&&, Group<Key, Value>> {
public: public:
...@@ -82,6 +85,9 @@ class Group : public GenImpl<Value&&, Group<Key, Value>> { ...@@ -82,6 +85,9 @@ class Group : public GenImpl<Value&&, Group<Key, Value>> {
return true; return true;
} }
// GroupBy only takes in finite generators, so we only have finite groups
static constexpr bool infinite = false;
private: private:
Key key_; Key key_;
mutable VectorType values_; mutable VectorType values_;
...@@ -89,6 +95,12 @@ class Group : public GenImpl<Value&&, Group<Key, Value>> { ...@@ -89,6 +95,12 @@ class Group : public GenImpl<Value&&, Group<Key, Value>> {
namespace detail { namespace detail {
// Classes used for the implementation of Sources, Operators, and Sinks
/*
******************************* Sources ***************************************
*/
/* /*
* ReferencedSource - Generate values from an STL-like container using * ReferencedSource - Generate values from an STL-like container using
* iterators from .begin() until .end(). Value type defaults to the type of * iterators from .begin() until .end(). Value type defaults to the type of
...@@ -101,23 +113,22 @@ namespace detail { ...@@ -101,23 +113,22 @@ namespace detail {
* string& longestName = from(names) * string& longestName = from(names)
* | maxBy([](string& s) { return s.size() }); * | maxBy([](string& s) { return s.size() });
*/ */
template<class Container, template <class Container, class Value>
class Value> class ReferencedSource
class ReferencedSource : : public GenImpl<Value, ReferencedSource<Container, Value>> {
public GenImpl<Value, ReferencedSource<Container, Value>> {
Container* container_; Container* container_;
public:
explicit ReferencedSource(Container* container)
: container_(container) {}
template<class Body> public:
explicit ReferencedSource(Container* container) : container_(container) {}
template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
for (auto& value : *container_) { for (auto& value : *container_) {
body(std::forward<Value>(value)); body(std::forward<Value>(value));
} }
} }
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
for (auto& value : *container_) { for (auto& value : *container_) {
if (!handler(std::forward<Value>(value))) { if (!handler(std::forward<Value>(value))) {
...@@ -126,6 +137,9 @@ public: ...@@ -126,6 +137,9 @@ public:
} }
return true; return true;
} }
// from takes in a normal stl structure, which are all finite
static constexpr bool infinite = false;
}; };
/** /**
...@@ -142,45 +156,44 @@ public: ...@@ -142,45 +156,44 @@ public:
* *
* Though it is also used for the initializer_list specialization of from(). * Though it is also used for the initializer_list specialization of from().
*/ */
template<class StorageType, template <class StorageType, class Container>
class Container> class CopiedSource
class CopiedSource : : public GenImpl<const StorageType&, CopiedSource<StorageType, Container>> {
public GenImpl<const StorageType&, static_assert(!std::is_reference<StorageType>::value,
CopiedSource<StorageType, Container>> { "StorageType must be decayed");
static_assert(
!std::is_reference<StorageType>::value, "StorageType must be decayed");
public: public:
// Generator objects are often copied during normal construction as they are // Generator objects are often copied during normal construction as they are
// encapsulated by downstream generators. It would be bad if this caused // encapsulated by downstream generators. It would be bad if this caused
// a copy of the entire container each time, and since we're only exposing a // a copy of the entire container each time, and since we're only exposing a
// const reference to the value, it's safe to share it between multiple // const reference to the value, it's safe to share it between multiple
// generators. // generators.
static_assert( static_assert(!std::is_reference<Container>::value,
!std::is_reference<Container>::value, "Can't copy into a reference");
"Can't copy into a reference");
std::shared_ptr<const Container> copy_; std::shared_ptr<const Container> copy_;
public:
public:
typedef Container ContainerType; typedef Container ContainerType;
template<class SourceContainer> template <class SourceContainer>
explicit CopiedSource(const SourceContainer& container) explicit CopiedSource(const SourceContainer& container)
: copy_(new Container(begin(container), end(container))) {} : copy_(new Container(begin(container), end(container))) {}
explicit CopiedSource(Container&& container) : explicit CopiedSource(Container&& container)
copy_(new Container(std::move(container))) {} : copy_(new Container(std::move(container))) {}
// To enable re-use of cached results. // To enable re-use of cached results.
CopiedSource(const CopiedSource<StorageType, Container>& source) CopiedSource(const CopiedSource<StorageType, Container>& source)
: copy_(source.copy_) {} : copy_(source.copy_) {}
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
for (const auto& value : *copy_) { for (const auto& value : *copy_) {
body(value); body(value);
} }
} }
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
// The collection may be reused by others, we can't allow it to be changed. // The collection may be reused by others, we can't allow it to be changed.
for (const auto& value : *copy_) { for (const auto& value : *copy_) {
...@@ -190,6 +203,9 @@ public: ...@@ -190,6 +203,9 @@ public:
} }
return true; return true;
} }
// from takes in a normal stl structure, which are all finite
static constexpr bool infinite = false;
}; };
/** /**
...@@ -203,17 +219,16 @@ public: ...@@ -203,17 +219,16 @@ public:
* *
* Reminder: Be careful not to invalidate iterators when using ranges like this. * Reminder: Be careful not to invalidate iterators when using ranges like this.
*/ */
template<class Iterator> template <class Iterator>
class RangeSource : public GenImpl<typename Range<Iterator>::reference, class RangeSource : public GenImpl<typename Range<Iterator>::reference,
RangeSource<Iterator>> { RangeSource<Iterator>> {
Range<Iterator> range_; Range<Iterator> range_;
public: public:
RangeSource() = default; RangeSource() = default;
explicit RangeSource(Range<Iterator> range) explicit RangeSource(Range<Iterator> range) : range_(std::move(range)) {}
: range_(std::move(range))
{}
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
for (auto& value : range_) { for (auto& value : range_) {
if (!handler(value)) { if (!handler(value)) {
...@@ -223,12 +238,15 @@ class RangeSource : public GenImpl<typename Range<Iterator>::reference, ...@@ -223,12 +238,15 @@ class RangeSource : public GenImpl<typename Range<Iterator>::reference,
return true; return true;
} }
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
for (auto& value : range_) { for (auto& value : range_) {
body(value); body(value);
} }
} }
// folly::Range only supports finite ranges
static constexpr bool infinite = false;
}; };
/** /**
...@@ -242,17 +260,19 @@ class RangeSource : public GenImpl<typename Range<Iterator>::reference, ...@@ -242,17 +260,19 @@ class RangeSource : public GenImpl<typename Range<Iterator>::reference,
* auto indexes = range(0, 10); * auto indexes = range(0, 10);
* auto endless = seq(0); // 0, 1, 2, 3, ... * auto endless = seq(0); // 0, 1, 2, 3, ...
*/ */
template<class Value, class SequenceImpl> template <class Value, class SequenceImpl>
class Sequence : public GenImpl<const Value&, Sequence<Value, SequenceImpl>> { class Sequence : public GenImpl<const Value&, Sequence<Value, SequenceImpl>> {
static_assert(!std::is_reference<Value>::value && static_assert(!std::is_reference<Value>::value &&
!std::is_const<Value>::value, "Value mustn't be const or ref."); !std::is_const<Value>::value,
"Value mustn't be const or ref.");
Value start_; Value start_;
SequenceImpl impl_; SequenceImpl impl_;
public:
public:
explicit Sequence(Value start, SequenceImpl impl) explicit Sequence(Value start, SequenceImpl impl)
: start_(std::move(start)), impl_(std::move(impl)) { } : start_(std::move(start)), impl_(std::move(impl)) {}
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
for (Value current = start_; impl_.test(current); impl_.step(current)) { for (Value current = start_; impl_.test(current); impl_.step(current)) {
if (!handler(current)) { if (!handler(current)) {
...@@ -262,71 +282,83 @@ public: ...@@ -262,71 +282,83 @@ public:
return true; return true;
} }
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
for (Value current = start_; impl_.test(current); impl_.step(current)) { for (Value current = start_; impl_.test(current); impl_.step(current)) {
body(current); body(current);
} }
} }
// Let the implementation say if we are infinite or not
static constexpr bool infinite = SequenceImpl::infinite;
}; };
/** /**
* Sequence implementations (range, sequence, infinite, with/without step) * Sequence implementations (range, sequence, infinite, with/without step)
**/ **/
template<class Value> template <class Value>
class RangeImpl { class RangeImpl {
Value end_; Value end_;
public: public:
explicit RangeImpl(Value end) : end_(std::move(end)) { } explicit RangeImpl(Value end) : end_(std::move(end)) {}
bool test(const Value& current) const { return current < end_; } bool test(const Value& current) const { return current < end_; }
void step(Value& current) const { ++current; } void step(Value& current) const { ++current; }
static constexpr bool infinite = false;
}; };
template<class Value, class Distance> template <class Value, class Distance>
class RangeWithStepImpl { class RangeWithStepImpl {
Value end_; Value end_;
Distance step_; Distance step_;
public: public:
explicit RangeWithStepImpl(Value end, Distance step) explicit RangeWithStepImpl(Value end, Distance step)
: end_(std::move(end)), step_(std::move(step)) { } : end_(std::move(end)), step_(std::move(step)) {}
bool test(const Value& current) const { return current < end_; } bool test(const Value& current) const { return current < end_; }
void step(Value& current) const { current += step_; } void step(Value& current) const { current += step_; }
static constexpr bool infinite = false;
}; };
template<class Value> template <class Value>
class SeqImpl { class SeqImpl {
Value end_; Value end_;
public: public:
explicit SeqImpl(Value end) : end_(std::move(end)) { } explicit SeqImpl(Value end) : end_(std::move(end)) {}
bool test(const Value& current) const { return current <= end_; } bool test(const Value& current) const { return current <= end_; }
void step(Value& current) const { ++current; } void step(Value& current) const { ++current; }
static constexpr bool infinite = false;
}; };
template<class Value, class Distance> template <class Value, class Distance>
class SeqWithStepImpl { class SeqWithStepImpl {
Value end_; Value end_;
Distance step_; Distance step_;
public: public:
explicit SeqWithStepImpl(Value end, Distance step) explicit SeqWithStepImpl(Value end, Distance step)
: end_(std::move(end)), step_(std::move(step)) { } : end_(std::move(end)), step_(std::move(step)) {}
bool test(const Value& current) const { return current <= end_; } bool test(const Value& current) const { return current <= end_; }
void step(Value& current) const { current += step_; } void step(Value& current) const { current += step_; }
static constexpr bool infinite = false;
}; };
template<class Value> template <class Value>
class InfiniteImpl { class InfiniteImpl {
public: public:
bool test(const Value& current) const { return true; } bool test(const Value& current) const { return true; }
void step(Value& current) const { ++current; } void step(Value& current) const { ++current; }
static constexpr bool infinite = true;
}; };
/** /**
* GenratorBuilder - Helper for GENERTATOR macro. * GenratorBuilder - Helper for GENERTATOR macro.
**/ **/
template<class Value> template <class Value>
struct GeneratorBuilder { struct GeneratorBuilder {
template<class Source, template <class Source,
class Yield = detail::Yield<Value, Source>> class Yield = detail::Yield<Value, Source>>
Yield operator+(Source&& source) { Yield operator+(Source&& source) {
return Yield(std::forward<Source>(source)); return Yield(std::forward<Source>(source));
} }
...@@ -336,15 +368,14 @@ struct GeneratorBuilder { ...@@ -336,15 +368,14 @@ struct GeneratorBuilder {
* Yield - For producing values from a user-defined generator by way of a * Yield - For producing values from a user-defined generator by way of a
* 'yield' function. * 'yield' function.
**/ **/
template<class Value, class Source> template <class Value, class Source>
class Yield : public GenImpl<Value, Yield<Value, Source>> { class Yield : public GenImpl<Value, Yield<Value, Source>> {
Source source_; Source source_;
public: public:
explicit Yield(Source source) explicit Yield(Source source) : source_(std::move(source)) {}
: source_(std::move(source)) {
}
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
struct Break {}; struct Break {};
auto body = [&](Value value) { auto body = [&](Value value) {
...@@ -360,13 +391,13 @@ class Yield : public GenImpl<Value, Yield<Value, Source>> { ...@@ -360,13 +391,13 @@ class Yield : public GenImpl<Value, Yield<Value, Source>> {
} }
} }
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
source_(std::forward<Body>(body)); source_(std::forward<Body>(body));
} }
}; };
template<class Value> template <class Value>
class Empty : public GenImpl<Value, Empty<Value>> { class Empty : public GenImpl<Value, Empty<Value>> {
public: public:
template <class Handler> template <class Handler>
...@@ -376,6 +407,9 @@ class Empty : public GenImpl<Value, Empty<Value>> { ...@@ -376,6 +407,9 @@ class Empty : public GenImpl<Value, Empty<Value>> {
template <class Body> template <class Body>
void foreach(Body&&) const {} void foreach(Body&&) const {}
// No values, so finite
static constexpr bool infinite = false;
}; };
template <class Value> template <class Value>
...@@ -383,6 +417,7 @@ class SingleReference : public GenImpl<Value&, SingleReference<Value>> { ...@@ -383,6 +417,7 @@ class SingleReference : public GenImpl<Value&, SingleReference<Value>> {
static_assert(!std::is_reference<Value>::value, static_assert(!std::is_reference<Value>::value,
"SingleReference requires non-ref types"); "SingleReference requires non-ref types");
Value* ptr_; Value* ptr_;
public: public:
explicit SingleReference(Value& ref) : ptr_(&ref) {} explicit SingleReference(Value& ref) : ptr_(&ref) {}
...@@ -395,6 +430,9 @@ class SingleReference : public GenImpl<Value&, SingleReference<Value>> { ...@@ -395,6 +430,9 @@ class SingleReference : public GenImpl<Value&, SingleReference<Value>> {
void foreach(Body&& body) const { void foreach(Body&& body) const {
body(*ptr_); body(*ptr_);
} }
// One value, so finite
static constexpr bool infinite = false;
}; };
template <class Value> template <class Value>
...@@ -402,6 +440,7 @@ class SingleCopy : public GenImpl<const Value&, SingleCopy<Value>> { ...@@ -402,6 +440,7 @@ class SingleCopy : public GenImpl<const Value&, SingleCopy<Value>> {
static_assert(!std::is_reference<Value>::value, static_assert(!std::is_reference<Value>::value,
"SingleCopy requires non-ref types"); "SingleCopy requires non-ref types");
Value value_; Value value_;
public: public:
explicit SingleCopy(Value value) : value_(std::forward<Value>(value)) {} explicit SingleCopy(Value value) : value_(std::forward<Value>(value)) {}
...@@ -414,10 +453,13 @@ class SingleCopy : public GenImpl<const Value&, SingleCopy<Value>> { ...@@ -414,10 +453,13 @@ class SingleCopy : public GenImpl<const Value&, SingleCopy<Value>> {
void foreach(Body&& body) const { void foreach(Body&& body) const {
body(value_); body(value_);
} }
// One value, so finite
static constexpr bool infinite = false;
}; };
/* /*
* Operators ***************************** Operators ***************************************
*/ */
/** /**
...@@ -428,37 +470,34 @@ class SingleCopy : public GenImpl<const Value&, SingleCopy<Value>> { ...@@ -428,37 +470,34 @@ class SingleCopy : public GenImpl<const Value&, SingleCopy<Value>> {
* *
* auto squares = seq(1, 10) | map(square) | asVector; * auto squares = seq(1, 10) | map(square) | asVector;
*/ */
template<class Predicate> template <class Predicate>
class Map : public Operator<Map<Predicate>> { class Map : public Operator<Map<Predicate>> {
Predicate pred_; Predicate pred_;
public: public:
Map() = default; Map() = default;
explicit Map(Predicate pred) explicit Map(Predicate pred) : pred_(std::move(pred)) {}
: pred_(std::move(pred))
{ } template <class Value,
class Source,
template<class Value, class Result = typename ArgumentReference<
class Source, typename std::result_of<Predicate(Value)>::type>::type>
class Result = typename ArgumentReference< class Generator : public GenImpl<Result, Generator<Value, Source, Result>> {
typename std::result_of<Predicate(Value)>::type
>::type>
class Generator :
public GenImpl<Result, Generator<Value, Source, Result>> {
Source source_; Source source_;
Predicate pred_; Predicate pred_;
public:
public:
explicit Generator(Source source, const Predicate& pred) explicit Generator(Source source, const Predicate& pred)
: source_(std::move(source)), pred_(pred) {} : source_(std::move(source)), pred_(pred) {}
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
source_.foreach([&](Value value) { source_.foreach(
body(pred_(std::forward<Value>(value))); [&](Value value) { body(pred_(std::forward<Value>(value))); });
});
} }
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
return source_.apply([&](Value value) { return source_.apply([&](Value value) {
return handler(pred_(std::forward<Value>(value))); return handler(pred_(std::forward<Value>(value)));
...@@ -468,16 +507,16 @@ class Map : public Operator<Map<Predicate>> { ...@@ -468,16 +507,16 @@ class Map : public Operator<Map<Predicate>> {
static constexpr bool infinite = Source::infinite; static constexpr bool infinite = Source::infinite;
}; };
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), pred_); return Gen(std::move(source.self()), pred_);
} }
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), pred_); return Gen(source.self(), pred_);
} }
...@@ -500,25 +539,24 @@ class Map : public Operator<Map<Predicate>> { ...@@ -500,25 +539,24 @@ class Map : public Operator<Map<Predicate>> {
* *
* will give a vector of all the pointers != nullptr. * will give a vector of all the pointers != nullptr.
*/ */
template<class Predicate> template <class Predicate>
class Filter : public Operator<Filter<Predicate>> { class Filter : public Operator<Filter<Predicate>> {
Predicate pred_; Predicate pred_;
public: public:
Filter() = default; Filter() = default;
explicit Filter(Predicate pred) explicit Filter(Predicate pred) : pred_(std::move(pred)) {}
: pred_(std::move(pred))
{ }
template<class Value, template <class Value, class Source>
class Source>
class Generator : public GenImpl<Value, Generator<Value, Source>> { class Generator : public GenImpl<Value, Generator<Value, Source>> {
Source source_; Source source_;
Predicate pred_; Predicate pred_;
public:
public:
explicit Generator(Source source, const Predicate& pred) explicit Generator(Source source, const Predicate& pred)
: source_(std::move(source)), pred_(pred) {} : source_(std::move(source)), pred_(pred) {}
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
source_.foreach([&](Value value) { source_.foreach([&](Value value) {
if (pred_(std::forward<Value>(value))) { if (pred_(std::forward<Value>(value))) {
...@@ -527,7 +565,7 @@ class Filter : public Operator<Filter<Predicate>> { ...@@ -527,7 +565,7 @@ class Filter : public Operator<Filter<Predicate>> {
}); });
} }
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
return source_.apply([&](Value value) -> bool { return source_.apply([&](Value value) -> bool {
if (pred_(std::forward<Value>(value))) { if (pred_(std::forward<Value>(value))) {
...@@ -540,16 +578,16 @@ class Filter : public Operator<Filter<Predicate>> { ...@@ -540,16 +578,16 @@ class Filter : public Operator<Filter<Predicate>> {
static constexpr bool infinite = Source::infinite; static constexpr bool infinite = Source::infinite;
}; };
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), pred_); return Gen(std::move(source.self()), pred_);
} }
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), pred_); return Gen(source.self(), pred_);
} }
...@@ -564,25 +602,24 @@ class Filter : public Operator<Filter<Predicate>> { ...@@ -564,25 +602,24 @@ class Filter : public Operator<Filter<Predicate>> {
* | until([](Item& item) { return item.score > 100; }) * | until([](Item& item) { return item.score > 100; })
* | asVector; * | asVector;
*/ */
template<class Predicate> template <class Predicate>
class Until : public Operator<Until<Predicate>> { class Until : public Operator<Until<Predicate>> {
Predicate pred_; Predicate pred_;
public: public:
Until() = default; Until() = default;
explicit Until(Predicate pred) explicit Until(Predicate pred) : pred_(std::move(pred)) {}
: pred_(std::move(pred))
{}
template<class Value, template <class Value, class Source>
class Source>
class Generator : public GenImpl<Value, Generator<Value, Source>> { class Generator : public GenImpl<Value, Generator<Value, Source>> {
Source source_; Source source_;
Predicate pred_; Predicate pred_;
public: public:
explicit Generator(Source source, const Predicate& pred) explicit Generator(Source source, const Predicate& pred)
: source_(std::move(source)), pred_(pred) {} : source_(std::move(source)), pred_(pred) {}
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
bool cancelled = false; bool cancelled = false;
source_.apply([&](Value value) -> bool { source_.apply([&](Value value) -> bool {
...@@ -597,24 +634,24 @@ class Until : public Operator<Until<Predicate>> { ...@@ -597,24 +634,24 @@ class Until : public Operator<Until<Predicate>> {
}); });
return !cancelled; return !cancelled;
} }
// Theoretically an 'until' might stop an infinite
static constexpr bool infinite = false;
}; };
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), pred_); return Gen(std::move(source.self()), pred_);
} }
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), pred_); return Gen(source.self(), pred_);
} }
// Theoretically an 'until' might stop an infinite
static constexpr bool infinite = false;
}; };
/** /**
...@@ -628,23 +665,24 @@ class Until : public Operator<Until<Predicate>> { ...@@ -628,23 +665,24 @@ class Until : public Operator<Until<Predicate>> {
*/ */
class Take : public Operator<Take> { class Take : public Operator<Take> {
size_t count_; size_t count_;
public: public:
explicit Take(size_t count) explicit Take(size_t count) : count_(count) {}
: count_(count) {}
template<class Value, template <class Value, class Source>
class Source> class Generator : public GenImpl<Value, Generator<Value, Source>> {
class Generator :
public GenImpl<Value, Generator<Value, Source>> {
Source source_; Source source_;
size_t count_; size_t count_;
public:
public:
explicit Generator(Source source, size_t count) explicit Generator(Source source, size_t count)
: source_(std::move(source)) , count_(count) {} : source_(std::move(source)), count_(count) {}
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
if (count_ == 0) { return false; } if (count_ == 0) {
return false;
}
size_t n = count_; size_t n = count_;
bool cancelled = false; bool cancelled = false;
source_.apply([&](Value value) -> bool { source_.apply([&](Value value) -> bool {
...@@ -656,18 +694,21 @@ class Take : public Operator<Take> { ...@@ -656,18 +694,21 @@ class Take : public Operator<Take> {
}); });
return !cancelled; return !cancelled;
} }
// take will stop an infinite generator
static constexpr bool infinite = false;
}; };
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), count_); return Gen(std::move(source.self()), count_);
} }
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), count_); return Gen(source.self(), count_);
} }
...@@ -695,42 +736,50 @@ class Stride : public Operator<Stride> { ...@@ -695,42 +736,50 @@ class Stride : public Operator<Stride> {
class Generator : public GenImpl<Value, Generator<Value, Source>> { class Generator : public GenImpl<Value, Generator<Value, Source>> {
Source source_; Source source_;
size_t stride_; size_t stride_;
public:
Generator(Source source, size_t stride) public:
: source_(std::move(source)), stride_(stride) {} explicit Generator(Source source, size_t stride)
: source_(std::move(source)), stride_(stride) {}
template <class Handler>
bool apply(Handler&& handler) const { template <class Handler>
size_t distance = stride_; bool apply(Handler&& handler) const {
return source_.apply([&](Value value)->bool { size_t distance = stride_;
if (++distance >= stride_) { return source_.apply([&](Value value) -> bool {
if (!handler(std::forward<Value>(value))) { if (++distance >= stride_) {
return false; if (!handler(std::forward<Value>(value))) {
} return false;
distance = 0; }
} distance = 0;
return true; }
}); return true;
} });
}
template <class Body>
void foreach(Body&& body) const { template <class Body>
size_t distance = stride_; void foreach(Body&& body) const {
source_.foreach([&](Value value) { size_t distance = stride_;
if (++distance >= stride_) { source_.foreach([&](Value value) {
body(std::forward<Value>(value)); if (++distance >= stride_) {
distance = 0; body(std::forward<Value>(value));
} distance = 0;
}); }
} });
}
// Taking every Nth of an infinite list is still infinte
static constexpr bool infinite = Source::infinite;
}; };
template <class Source, class Value, class Gen = Generator<Value, Source>> template <class Source,
class Value,
class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), stride_); return Gen(std::move(source.self()), stride_);
} }
template <class Source, class Value, class Gen = Generator<Value, Source>> template <class Source,
class Value,
class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), stride_); return Gen(source.self(), stride_);
} }
...@@ -740,21 +789,22 @@ class Stride : public Operator<Stride> { ...@@ -740,21 +789,22 @@ class Stride : public Operator<Stride> {
* Sample - For taking a random sample of N elements from a sequence * Sample - For taking a random sample of N elements from a sequence
* (without replacement). * (without replacement).
*/ */
template<class Random> template <class Random>
class Sample : public Operator<Sample<Random>> { class Sample : public Operator<Sample<Random>> {
size_t count_; size_t count_;
Random rng_; Random rng_;
public: public:
explicit Sample(size_t count, Random rng) explicit Sample(size_t count, Random rng)
: count_(count), rng_(std::move(rng)) {} : count_(count), rng_(std::move(rng)) {}
template<class Value, template <class Value,
class Source, class Source,
class Rand, class Rand,
class StorageType = typename std::decay<Value>::type> class StorageType = typename std::decay<Value>::type>
class Generator : class Generator
public GenImpl<StorageType&&, : public GenImpl<StorageType&&,
Generator<Value, Source, Rand, StorageType>> { Generator<Value, Source, Rand, StorageType>> {
static_assert(!Source::infinite, "Cannot sample infinite source!"); static_assert(!Source::infinite, "Cannot sample infinite source!");
// It's too easy to bite ourselves if random generator is only 16-bit // It's too easy to bite ourselves if random generator is only 16-bit
static_assert(Random::max() >= std::numeric_limits<int32_t>::max() - 1, static_assert(Random::max() >= std::numeric_limits<int32_t>::max() - 1,
...@@ -762,53 +812,59 @@ class Sample : public Operator<Sample<Random>> { ...@@ -762,53 +812,59 @@ class Sample : public Operator<Sample<Random>> {
Source source_; Source source_;
size_t count_; size_t count_;
mutable Rand rng_; mutable Rand rng_;
public:
public:
explicit Generator(Source source, size_t count, Random rng) explicit Generator(Source source, size_t count, Random rng)
: source_(std::move(source)) , count_(count), rng_(std::move(rng)) {} : source_(std::move(source)), count_(count), rng_(std::move(rng)) {}
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
if (count_ == 0) { return false; } if (count_ == 0) {
return false;
}
std::vector<StorageType> v; std::vector<StorageType> v;
v.reserve(count_); v.reserve(count_);
// use reservoir sampling to give each source value an equal chance // use reservoir sampling to give each source value an equal chance
// of appearing in our output. // of appearing in our output.
size_t n = 1; size_t n = 1;
source_.foreach([&](Value value) -> void { source_.foreach([&](Value value) -> void {
if (v.size() < count_) { if (v.size() < count_) {
v.push_back(std::forward<Value>(value)); v.push_back(std::forward<Value>(value));
} else { } else {
// alternatively, we could create a std::uniform_int_distribution // alternatively, we could create a std::uniform_int_distribution
// instead of using modulus, but benchmarks show this has // instead of using modulus, but benchmarks show this has
// substantial overhead. // substantial overhead.
size_t index = rng_() % n; size_t index = rng_() % n;
if (index < v.size()) { if (index < v.size()) {
v[index] = std::forward<Value>(value); v[index] = std::forward<Value>(value);
}
} }
++n; }
}); ++n;
});
// output is unsorted! // output is unsorted!
for (auto& val: v) { for (auto& val : v) {
if (!handler(std::move(val))) { if (!handler(std::move(val))) {
return false; return false;
} }
} }
return true; return true;
} }
// Only takes N elements, so finite
static constexpr bool infinite = false;
}; };
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source, Random>> class Gen = Generator<Value, Source, Random>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), count_, rng_); return Gen(std::move(source.self()), count_, rng_);
} }
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source, Random>> class Gen = Generator<Value, Source, Random>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), count_, rng_); return Gen(source.self(), count_, rng_);
} }
...@@ -825,21 +881,20 @@ class Sample : public Operator<Sample<Random>> { ...@@ -825,21 +881,20 @@ class Sample : public Operator<Sample<Random>> {
*/ */
class Skip : public Operator<Skip> { class Skip : public Operator<Skip> {
size_t count_; size_t count_;
public: public:
explicit Skip(size_t count) explicit Skip(size_t count) : count_(count) {}
: count_(count) {}
template<class Value, template <class Value, class Source>
class Source> class Generator : public GenImpl<Value, Generator<Value, Source>> {
class Generator :
public GenImpl<Value, Generator<Value, Source>> {
Source source_; Source source_;
size_t count_; size_t count_;
public: public:
explicit Generator(Source source, size_t count) explicit Generator(Source source, size_t count)
: source_(std::move(source)) , count_(count) {} : source_(std::move(source)), count_(count) {}
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
if (count_ == 0) { if (count_ == 0) {
source_.foreach(body); source_.foreach(body);
...@@ -847,42 +902,43 @@ class Skip : public Operator<Skip> { ...@@ -847,42 +902,43 @@ class Skip : public Operator<Skip> {
} }
size_t n = 0; size_t n = 0;
source_.foreach([&](Value value) { source_.foreach([&](Value value) {
if (n < count_) { if (n < count_) {
++n; ++n;
} else { } else {
body(std::forward<Value>(value)); body(std::forward<Value>(value));
} }
}); });
} }
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
if (count_ == 0) { if (count_ == 0) {
return source_.apply(std::forward<Handler>(handler)); return source_.apply(std::forward<Handler>(handler));
} }
size_t n = 0; size_t n = 0;
return source_.apply([&](Value value) -> bool { return source_.apply([&](Value value) -> bool {
if (n < count_) { if (n < count_) {
++n; ++n;
return true; return true;
} }
return handler(std::forward<Value>(value)); return handler(std::forward<Value>(value));
}); });
} }
// Skipping N items of an infinite source is still infinite
static constexpr bool infinite = Source::infinite; static constexpr bool infinite = Source::infinite;
}; };
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), count_); return Gen(std::move(source.self()), count_);
} }
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), count_); return Gen(source.self(), count_);
} }
...@@ -901,30 +957,26 @@ class Skip : public Operator<Skip> { ...@@ -901,30 +957,26 @@ class Skip : public Operator<Skip> {
* }) * })
* | take(10); * | take(10);
*/ */
template<class Selector, class Comparer> template <class Selector, class Comparer>
class Order : public Operator<Order<Selector, Comparer>> { class Order : public Operator<Order<Selector, Comparer>> {
Selector selector_; Selector selector_;
Comparer comparer_; Comparer comparer_;
public: public:
Order() = default; Order() = default;
explicit Order(Selector selector) explicit Order(Selector selector) : selector_(std::move(selector)) {}
: selector_(std::move(selector))
{} Order(Selector selector, Comparer comparer)
: selector_(std::move(selector)), comparer_(std::move(comparer)) {}
Order(Selector selector,
Comparer comparer) template <class Value,
: selector_(std::move(selector)) class Source,
, comparer_(std::move(comparer)) class StorageType = typename std::decay<Value>::type,
{} class Result = typename std::result_of<Selector(Value)>::type>
class Generator
template<class Value, : public GenImpl<StorageType&&,
class Source, Generator<Value, Source, StorageType, Result>> {
class StorageType = typename std::decay<Value>::type,
class Result = typename std::result_of<Selector(Value)>::type>
class Generator :
public GenImpl<StorageType&&,
Generator<Value, Source, StorageType, Result>> {
static_assert(!Source::infinite, "Cannot sort infinite source!"); static_assert(!Source::infinite, "Cannot sort infinite source!");
Source source_; Source source_;
Selector selector_; Selector selector_;
...@@ -940,13 +992,12 @@ class Order : public Operator<Order<Selector, Comparer>> { ...@@ -940,13 +992,12 @@ class Order : public Operator<Order<Selector, Comparer>> {
std::sort(vals.begin(), vals.end(), comparer); std::sort(vals.begin(), vals.end(), comparer);
return std::move(vals); return std::move(vals);
} }
public: public:
Generator(Source source, Generator(Source source, Selector selector, Comparer comparer)
Selector selector, : source_(std::move(source)),
Comparer comparer) selector_(std::move(selector)),
: source_(std::move(source)), comparer_(std::move(comparer)) {}
selector_(std::move(selector)),
comparer_(std::move(comparer)) {}
VectorType operator|(const Collect<VectorType>&) const { VectorType operator|(const Collect<VectorType>&) const {
return asVector(); return asVector();
...@@ -956,14 +1007,14 @@ class Order : public Operator<Order<Selector, Comparer>> { ...@@ -956,14 +1007,14 @@ class Order : public Operator<Order<Selector, Comparer>> {
return asVector(); return asVector();
} }
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
for (auto& value : asVector()) { for (auto& value : asVector()) {
body(std::move(value)); body(std::move(value));
} }
} }
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
auto comparer = [&](const StorageType& a, const StorageType& b) { auto comparer = [&](const StorageType& a, const StorageType& b) {
// swapped for minHeap // swapped for minHeap
...@@ -980,18 +1031,21 @@ class Order : public Operator<Order<Selector, Comparer>> { ...@@ -980,18 +1031,21 @@ class Order : public Operator<Order<Selector, Comparer>> {
} }
return true; return true;
} }
// Can only be run on and produce finite generators
static constexpr bool infinite = false;
}; };
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), selector_, comparer_); return Gen(std::move(source.self()), selector_, comparer_);
} }
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), selector_, comparer_); return Gen(source.self(), selector_, comparer_);
} }
...@@ -1057,14 +1111,21 @@ class GroupBy : public Operator<GroupBy<Selector>> { ...@@ -1057,14 +1111,21 @@ class GroupBy : public Operator<GroupBy<Selector>> {
} }
return true; return true;
} }
// Can only be run on and produce finite generators
static constexpr bool infinite = false;
}; };
template <class Source, class Value, class Gen = Generator<Value, Source>> template <class Source,
class Value,
class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), selector_); return Gen(std::move(source.self()), selector_);
} }
template <class Source, class Value, class Gen = Generator<Value, Source>> template <class Source,
class Value,
class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), selector_); return Gen(source.self(), selector_);
} }
...@@ -1079,17 +1140,17 @@ class GroupBy : public Operator<GroupBy<Selector>> { ...@@ -1079,17 +1140,17 @@ class GroupBy : public Operator<GroupBy<Selector>> {
* auto c = from(vector) | assert_type<int&>() | sum; * auto c = from(vector) | assert_type<int&>() | sum;
* *
*/ */
template<class Expected> template <class Expected>
class TypeAssertion : public Operator<TypeAssertion<Expected>> { class TypeAssertion : public Operator<TypeAssertion<Expected>> {
public: public:
template<class Source, class Value> template <class Source, class Value>
const Source& compose(const GenImpl<Value, Source>& source) const { const Source& compose(const GenImpl<Value, Source>& source) const {
static_assert(std::is_same<Expected, Value>::value, static_assert(std::is_same<Expected, Value>::value,
"assert_type() check failed"); "assert_type() check failed");
return source.self(); return source.self();
} }
template<class Source, class Value> template <class Source, class Value>
Source&& compose(GenImpl<Value, Source>&& source) const { Source&& compose(GenImpl<Value, Source>&& source) const {
static_assert(std::is_same<Expected, Value>::value, static_assert(std::is_same<Expected, Value>::value,
"assert_type() check failed"); "assert_type() check failed");
...@@ -1109,18 +1170,16 @@ class TypeAssertion : public Operator<TypeAssertion<Expected>> { ...@@ -1109,18 +1170,16 @@ class TypeAssertion : public Operator<TypeAssertion<Expected>> {
* }) * })
* | take(10); * | take(10);
*/ */
template<class Selector> template <class Selector>
class Distinct : public Operator<Distinct<Selector>> { class Distinct : public Operator<Distinct<Selector>> {
Selector selector_; Selector selector_;
public: public:
Distinct() = default; Distinct() = default;
explicit Distinct(Selector selector) explicit Distinct(Selector selector) : selector_(std::move(selector)) {}
: selector_(std::move(selector))
{}
template<class Value, template <class Value, class Source>
class Source>
class Generator : public GenImpl<Value, Generator<Value, Source>> { class Generator : public GenImpl<Value, Generator<Value, Source>> {
Source source_; Source source_;
Selector selector_; Selector selector_;
...@@ -1135,12 +1194,10 @@ class Distinct : public Operator<Distinct<Selector>> { ...@@ -1135,12 +1194,10 @@ class Distinct : public Operator<Distinct<Selector>> {
typedef typename std::decay<KeyType>::type KeyStorageType; typedef typename std::decay<KeyType>::type KeyStorageType;
public: public:
Generator(Source source, Generator(Source source, Selector selector)
Selector selector) : source_(std::move(source)), selector_(std::move(selector)) {}
: source_(std::move(source)),
selector_(std::move(selector)) {}
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
std::unordered_set<KeyStorageType> keysSeen; std::unordered_set<KeyStorageType> keysSeen;
source_.foreach([&](Value value) { source_.foreach([&](Value value) {
...@@ -1150,7 +1207,7 @@ class Distinct : public Operator<Distinct<Selector>> { ...@@ -1150,7 +1207,7 @@ class Distinct : public Operator<Distinct<Selector>> {
}); });
} }
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
std::unordered_set<KeyStorageType> keysSeen; std::unordered_set<KeyStorageType> keysSeen;
return source_.apply([&](Value value) -> bool { return source_.apply([&](Value value) -> bool {
...@@ -1160,18 +1217,22 @@ class Distinct : public Operator<Distinct<Selector>> { ...@@ -1160,18 +1217,22 @@ class Distinct : public Operator<Distinct<Selector>> {
return true; return true;
}); });
} }
// While running distinct on an infinite sequence might produce a
// conceptually finite sequence, it will take infinite time
static constexpr bool infinite = Source::infinite;
}; };
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), selector_); return Gen(std::move(source.self()), selector_);
} }
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), selector_); return Gen(source.self(), selector_);
} }
...@@ -1181,16 +1242,16 @@ class Distinct : public Operator<Distinct<Selector>> { ...@@ -1181,16 +1242,16 @@ class Distinct : public Operator<Distinct<Selector>> {
* Composer - Helper class for adapting pipelines into functors. Primarily used * Composer - Helper class for adapting pipelines into functors. Primarily used
* for 'mapOp'. * for 'mapOp'.
*/ */
template<class Operators> template <class Operators>
class Composer { class Composer {
Operators op_; Operators op_;
public: public:
explicit Composer(Operators op) explicit Composer(Operators op) : op_(std::move(op)) {}
: op_(std::move(op)) {}
template<class Source, template <class Source,
class Ret = decltype(std::declval<Operators>() class Ret = decltype(
.compose(std::declval<Source>()))> std::declval<Operators>().compose(std::declval<Source>()))>
Ret operator()(Source&& source) const { Ret operator()(Source&& source) const {
return op_.compose(std::forward<Source>(source)); return op_.compose(std::forward<Source>(source));
} }
...@@ -1211,42 +1272,42 @@ class Composer { ...@@ -1211,42 +1272,42 @@ class Composer {
*/ */
class Batch : public Operator<Batch> { class Batch : public Operator<Batch> {
size_t batchSize_; size_t batchSize_;
public: public:
explicit Batch(size_t batchSize) explicit Batch(size_t batchSize) : batchSize_(batchSize) {
: batchSize_(batchSize) {
if (batchSize_ == 0) { if (batchSize_ == 0) {
throw std::invalid_argument("Batch size must be non-zero!"); throw std::invalid_argument("Batch size must be non-zero!");
} }
} }
template<class Value, template <class Value,
class Source, class Source,
class StorageType = typename std::decay<Value>::type, class StorageType = typename std::decay<Value>::type,
class VectorType = std::vector<StorageType>> class VectorType = std::vector<StorageType>>
class Generator : class Generator
public GenImpl<VectorType&, : public GenImpl<VectorType&,
Generator<Value, Source, StorageType, VectorType>> { Generator<Value, Source, StorageType, VectorType>> {
Source source_; Source source_;
size_t batchSize_; size_t batchSize_;
public:
public:
explicit Generator(Source source, size_t batchSize) explicit Generator(Source source, size_t batchSize)
: source_(std::move(source)) : source_(std::move(source)), batchSize_(batchSize) {}
, batchSize_(batchSize) {}
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
VectorType batch_; VectorType batch_;
batch_.reserve(batchSize_); batch_.reserve(batchSize_);
bool shouldContinue = source_.apply([&](Value value) -> bool { bool shouldContinue = source_.apply([&](Value value) -> bool {
batch_.push_back(std::forward<Value>(value)); batch_.push_back(std::forward<Value>(value));
if (batch_.size() == batchSize_) { if (batch_.size() == batchSize_) {
bool needMore = handler(batch_); bool needMore = handler(batch_);
batch_.clear(); batch_.clear();
return needMore; return needMore;
} }
// Always need more if the handler is not called. // Always need more if the handler is not called.
return true; return true;
}); });
// Flush everything, if and only if `handler` hasn't returned false. // Flush everything, if and only if `handler` hasn't returned false.
if (shouldContinue && !batch_.empty()) { if (shouldContinue && !batch_.empty()) {
shouldContinue = handler(batch_); shouldContinue = handler(batch_);
...@@ -1255,765 +1316,780 @@ class Batch : public Operator<Batch> { ...@@ -1255,765 +1316,780 @@ class Batch : public Operator<Batch> {
return shouldContinue; return shouldContinue;
} }
// Taking n-tuples of an infinite source is still infinite
static constexpr bool infinite = Source::infinite; static constexpr bool infinite = Source::infinite;
}; };
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), batchSize_); return Gen(std::move(source.self()), batchSize_);
} }
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), batchSize_); return Gen(source.self(), batchSize_);
} }
}; };
/*
* Sinks
*/
/** /**
* FoldLeft - Left-associative functional fold. For producing an aggregate value * Concat - For flattening generators of generators.
* from a seed and a folder function. Useful for custom aggregators on a
* sequence.
* *
* This type is primarily used through the 'foldl' helper method, like: * This type is usually used through the 'concat' static value, like:
* *
* double movingAverage = from(values) * auto edges =
* | foldl(0.0, [](double avg, double sample) { * from(nodes)
* return sample * 0.1 + avg * 0.9; * | map([](Node& x) {
* }); * return from(x.neighbors)
* | map([&](Node& y) {
* return Edge(x, y);
* });
* })
* | concat
* | as<std::set>();
*/ */
template<class Seed, class Concat : public Operator<Concat> {
class Fold>
class FoldLeft : public Operator<FoldLeft<Seed, Fold>> {
Seed seed_;
Fold fold_;
public: public:
FoldLeft() = default; Concat() = default;
FoldLeft(Seed seed,
Fold fold) template <class Inner,
: seed_(std::move(seed)) class Source,
, fold_(std::move(fold)) class InnerValue = typename std::decay<Inner>::type::ValueType>
{} class Generator
: public GenImpl<InnerValue, Generator<Inner, Source, InnerValue>> {
template<class Source, Source source_;
class Value>
Seed compose(const GenImpl<Value, Source>& source) const { public:
static_assert(!Source::infinite, "Cannot foldl infinite source"); explicit Generator(Source source) : source_(std::move(source)) {}
Seed accum = seed_;
source | [&](Value v) { template <class Handler>
accum = fold_(std::move(accum), std::forward<Value>(v)); bool apply(Handler&& handler) const {
}; return source_.apply([&](Inner inner) -> bool {
return accum; return inner.apply(std::forward<Handler>(handler));
});
}
template <class Body>
void foreach(Body&& body) const {
source_.foreach([&](Inner inner) {
inner.foreach(std::forward<Body>(body));
});
}
// Resulting concatination is only finite if both Source and Inner are also
// finite. In one sence, if dosn't make sence to call concat when the Inner
// generator is infinite (you could just call first), so we could also just
// static_assert if the inner is infinite. Taking the less restrictive
// approch for now.
static constexpr bool infinite =
Source::infinite || std::decay<Inner>::type::infinite;
};
template <class Value,
class Source,
class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()));
}
template <class Value,
class Source,
class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self());
} }
}; };
/** /**
* First - For finding the first value in a sequence. * RangeConcat - For flattening generators of iterables.
* *
* This type is primarily used through the 'first' static value, like: * This type is usually used through the 'rconcat' static value, like:
* *
* int firstThreeDigitPrime = seq(100) | filter(isPrime) | first; * map<int, vector<int>> adjacency;
* auto sinks =
* from(adjacency)
* | get<1>()
* | rconcat()
* | as<std::set>();
*/ */
class First : public Operator<First> { class RangeConcat : public Operator<RangeConcat> {
public: public:
First() = default; RangeConcat() = default;
template<class Source,
class Value,
class StorageType = typename std::decay<Value>::type>
StorageType compose(const GenImpl<Value, Source>& source) const {
Optional<StorageType> accum;
source | [&](Value v) -> bool {
accum = std::forward<Value>(v);
return false;
};
if (!accum.hasValue()) {
throw EmptySequence();
}
return std::move(accum.value());
}
};
/** template <class Range,
* IsEmpty - a helper class for isEmpty and notEmpty class Source,
* class InnerValue = typename ValueTypeOfRange<Range>::RefType>
* Essentially returns 'result' if the source is empty. Note that this cannot be class Generator
* called on an infinite source, because then there is only one possible return : public GenImpl<InnerValue, Generator<Range, Source, InnerValue>> {
* value. Source source_;
*/
template <bool emptyResult>
class IsEmpty : public Operator<IsEmpty<emptyResult>> {
public:
IsEmpty() = default;
template<class Source, public:
class Value> explicit Generator(Source source) : source_(std::move(source)) {}
bool compose(const GenImpl<Value, Source>& source) const {
static_assert(!Source::infinite,
"Cannot call 'all', 'any', 'isEmpty', or 'notEmpty' on "
"infinite source. 'all' and 'isEmpty' will either return "
"false or hang. 'any' or 'notEmpty' will either return true "
"or hang.");
bool ans = emptyResult;
source | [&](Value v) -> bool {
ans = !emptyResult;
return false;
};
return ans;
}
};
/** template <class Body>
* Reduce - Functional reduce, for recursively combining values from a source void foreach(Body&& body) const {
* using a reducer function until there is only one item left. Useful for source_.foreach([&](Range range) {
* combining values when an empty sequence doesn't make sense. for (auto& value : range) {
* body(value);
* This type is primarily used through the 'reduce' helper method, like: }
* });
* sring longest = from(names) }
* | reduce([](string&& best, string& current) {
* return best.size() >= current.size() ? best : current;
* });
*/
template<class Reducer>
class Reduce : public Operator<Reduce<Reducer>> {
Reducer reducer_;
public:
Reduce() = default;
explicit Reduce(Reducer reducer)
: reducer_(std::move(reducer))
{}
template<class Source, template <class Handler>
class Value, bool apply(Handler&& handler) const {
class StorageType = typename std::decay<Value>::type> return source_.apply([&](Range range) -> bool {
StorageType compose(const GenImpl<Value, Source>& source) const { for (auto& value : range) {
Optional<StorageType> accum; if (!handler(value)) {
source | [&](Value v) { return false;
if (accum.hasValue()) { }
accum = reducer_(std::move(accum.value()), std::forward<Value>(v)); }
} else { return true;
accum = std::forward<Value>(v); });
}
};
if (!accum.hasValue()) {
throw EmptySequence();
} }
return accum.value();
}
};
/** // This is similar to concat, except that the inner iterables all are finite
* Count - for simply counting the items in a collection. // so the only thing that matters is that the source is infinite.
* static constexpr bool infinite = Source::infinite;
* This type is usually used through its singleton, 'count': };
*
* auto shortPrimes = seq(1, 100) | filter(isPrime) | count;
*/
class Count : public Operator<Count> {
public:
Count() = default;
template<class Source, template <class Value,
class Value> class Source,
size_t compose(const GenImpl<Value, Source>& source) const { class Gen = Generator<Value, Source>>
static_assert(!Source::infinite, "Cannot count infinite source"); Gen compose(GenImpl<Value, Source>&& source) const {
return foldl(size_t(0), return Gen(std::move(source.self()));
[](size_t accum, Value v) {
return accum + 1;
}).compose(source);
} }
};
/** template <class Value,
* Sum - For simply summing up all the values from a source. class Source,
* class Gen = Generator<Value, Source>>
* This type is usually used through its singleton, 'sum': Gen compose(const GenImpl<Value, Source>& source) const {
* return Gen(source.self());
* auto gaussSum = seq(1, 100) | sum;
*/
class Sum : public Operator<Sum> {
public:
Sum() = default;
template<class Source,
class Value,
class StorageType = typename std::decay<Value>::type>
StorageType compose(const GenImpl<Value, Source>& source) const {
static_assert(!Source::infinite, "Cannot sum infinite source");
return foldl(StorageType(0),
[](StorageType&& accum, Value v) {
return std::move(accum) + std::forward<Value>(v);
}).compose(source);
} }
}; };
/** /**
* Contains - For testing whether a value matching the given value is contained * GuardImpl - For handling exceptions from downstream computation. Requires the
* in a sequence. * type of exception to catch, and handler function to invoke in the event of
* * the exception. Note that the handler may:
* This type should be used through the 'contains' helper method, like: * 1) return true to continue processing the sequence
* 2) return false to end the sequence immediately
* 3) throw, to pass the exception to the next catch
* The handler must match the signature 'bool(Exception&, Value)'.
* *
* bool contained = seq(1, 10) | map(square) | contains(49); * This type is used through the `guard` helper, like so:
*/
template<class Needle>
class Contains : public Operator<Contains<Needle>> {
Needle needle_;
public:
explicit Contains(Needle needle)
: needle_(std::move(needle))
{}
template<class Source,
class Value,
class StorageType = typename std::decay<Value>::type>
bool compose(const GenImpl<Value, Source>& source) const {
static_assert(!Source::infinite,
"Calling contains on an infinite source might cause "
"an infinite loop.");
return !(source | [this](Value value) {
return !(needle_ == std::forward<Value>(value));
});
}
};
/**
* Min - For a value which minimizes a key, where the key is determined by a
* given selector, and compared by given comparer.
* *
* This type is usually used through the singletone 'min' or through the helper * auto indexes
* functions 'minBy' and 'maxBy'. * = byLine(STDIN_FILENO)
* | guard<std::runtime_error>([](std::runtime_error& e,
* StringPiece sp) {
* LOG(ERROR) << sp << ": " << e.str();
* return true; // continue processing subsequent lines
* })
* | eachTo<int>()
* | as<vector>();
* *
* auto oldest = from(people) * TODO(tjackson): Rename this back to Guard.
* | minBy([](Person& p) { **/
* return p.dateOfBirth; template <class Exception, class ErrorHandler>
* }); class GuardImpl : public Operator<GuardImpl<Exception, ErrorHandler>> {
*/ ErrorHandler handler_;
template<class Selector,
class Comparer>
class Min : public Operator<Min<Selector, Comparer>> {
Selector selector_;
Comparer comparer_;
public:
Min() = default;
explicit Min(Selector selector)
: selector_(std::move(selector))
{}
Min(Selector selector,
Comparer comparer)
: selector_(std::move(selector))
, comparer_(std::move(comparer))
{}
template<class Value,
class Source,
class StorageType = typename std::decay<Value>::type,
class Key = typename std::decay<
typename std::result_of<Selector(Value)>::type
>::type>
StorageType compose(const GenImpl<Value, Source>& source) const {
Optional<StorageType> min;
Optional<Key> minKey;
source | [&](Value v) {
Key key = selector_(std::forward<Value>(v));
if (!minKey.hasValue() || comparer_(key, minKey.value())) {
minKey = key;
min = std::forward<Value>(v);
}
};
if (!min.hasValue()) {
throw EmptySequence();
}
return min.value();
}
};
/**
* Append - For collecting values from a source into a given output container
* by appending.
*
* This type is usually used through the helper function 'appendTo', like:
*
* vector<int64_t> ids;
* from(results) | map([](Person& p) { return p.id })
* | appendTo(ids);
*/
template<class Collection>
class Append : public Operator<Append<Collection>> {
Collection* collection_;
public: public:
explicit Append(Collection* collection) explicit GuardImpl(ErrorHandler handler) : handler_(std::move(handler)) {}
: collection_(collection)
{}
template<class Value, template <class Value, class Source>
class Source> class Generator : public GenImpl<Value, Generator<Value, Source>> {
Collection& compose(const GenImpl<Value, Source>& source) const { Source source_;
source | [&](Value v) { ErrorHandler handler_;
collection_->insert(collection_->end(), std::forward<Value>(v));
};
return *collection_;
}
};
/** public:
* Collect - For collecting values from a source in a collection of the desired explicit Generator(Source source, ErrorHandler handler)
* type. : source_(std::move(source)), handler_(std::move(handler)) {}
*
* This type is usually used through the helper function 'as', like:
*
* std::string upper = from(stringPiece)
* | map(&toupper)
* | as<std::string>();
*/
template<class Collection>
class Collect : public Operator<Collect<Collection>> {
public:
Collect() = default;
template<class Value, template <class Handler>
class Source, bool apply(Handler&& handler) const {
class StorageType = typename std::decay<Value>::type> return source_.apply([&](Value value) -> bool {
Collection compose(const GenImpl<Value, Source>& source) const { try {
Collection collection; handler(std::forward<Value>(value));
source | [&](Value v) { return true;
collection.insert(collection.end(), std::forward<Value>(v)); } catch (Exception& e) {
}; return handler_(e, std::forward<Value>(value));
return collection; }
} });
}; }
// Just passes value though, length unaffected
static constexpr bool infinite = Source::infinite;
};
/** template <class Value,
* CollectTemplate - For collecting values from a source in a collection class Source,
* constructed using the specified template type. Given the type of values class Gen = Generator<Value, Source>>
* produced by the given generator, the collection type will be: Gen compose(GenImpl<Value, Source>&& source) const {
* Container<Value, Allocator<Value>> return Gen(std::move(source.self()), handler_);
* }
* The allocator defaults to std::allocator, so this may be used for the STL
* containers by simply using operators like 'as<set>', 'as<deque>',
* 'as<vector>'. 'as', here is the helper method which is the usual means of
* consturcting this operator.
*
* Example:
*
* set<string> uniqueNames = from(names) | as<set>();
*/
template<template<class, class> class Container,
template<class> class Allocator>
class CollectTemplate : public Operator<CollectTemplate<Container, Allocator>> {
public:
CollectTemplate() = default;
template<class Value, template <class Value,
class Source, class Source,
class StorageType = typename std::decay<Value>::type, class Gen = Generator<Value, Source>>
class Collection = Container<StorageType, Allocator<StorageType>>> Gen compose(const GenImpl<Value, Source>& source) const {
Collection compose(const GenImpl<Value, Source>& source) const { return Gen(source.self(), handler_);
Collection collection;
source | [&](Value v) {
collection.insert(collection.end(), std::forward<Value>(v));
};
return collection;
} }
}; };
/** /**
* Concat - For flattening generators of generators. * Dereference - For dereferencing a sequence of pointers while filtering out
* null pointers.
* *
* This type is usually used through the 'concat' static value, like: * This type is usually used through the 'dereference' static value, like:
* *
* auto edges = * auto refs = from(ptrs) | dereference;
* from(nodes)
* | map([](Node& x) {
* return from(x.neighbors)
* | map([&](Node& y) {
* return Edge(x, y);
* });
* })
* | concat
* | as<std::set>();
*/ */
class Concat : public Operator<Concat> { class Dereference : public Operator<Dereference> {
public: public:
Concat() = default; Dereference() = default;
template<class Inner, template <class Value,
class Source, class Source,
class InnerValue = typename std::decay<Inner>::type::ValueType> class Result = decltype(*std::declval<Value>())>
class Generator : class Generator : public GenImpl<Result, Generator<Value, Source, Result>> {
public GenImpl<InnerValue, Generator<Inner, Source, InnerValue>> {
Source source_; Source source_;
public: public:
explicit Generator(Source source) explicit Generator(Source source) : source_(std::move(source)) {}
: source_(std::move(source)) {}
template<class Handler> template <class Body>
bool apply(Handler&& handler) const { void foreach(Body&& body) const {
return source_.apply([&](Inner inner) -> bool { source_.foreach([&](Value value) {
return inner.apply(std::forward<Handler>(handler)); if (value) {
}); return body(*value);
}
});
} }
template<class Body> template <class Handler>
void foreach(Body&& body) const { bool apply(Handler&& handler) const {
source_.foreach([&](Inner inner) { return source_.apply([&](Value value) -> bool {
inner.foreach(std::forward<Body>(body)); if (value) {
}); return handler(*value);
}
return true;
});
} }
// Just passes value though, length unaffected
static constexpr bool infinite = Source::infinite; static constexpr bool infinite = Source::infinite;
}; };
template<class Value, template <class Source,
class Source, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self())); return Gen(std::move(source.self()));
} }
template<class Value, template <class Source,
class Source, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self()); return Gen(source.self());
} }
}; };
/** /**
* RangeConcat - For flattening generators of iterables. * Indirect - For producing a sequence of the addresses of the values in the
* input.
* *
* This type is usually used through the 'rconcat' static value, like: * This type is usually used through the 'indirect' static value, like:
* *
* map<int, vector<int>> adjacency; * auto ptrs = from(refs) | indirect;
* auto sinks =
* from(adjacency)
* | get<1>()
* | rconcat()
* | as<std::set>();
*/ */
class RangeConcat : public Operator<RangeConcat> { class Indirect : public Operator<Indirect> {
public: public:
RangeConcat() = default; Indirect() = default;
template<class Range, template <class Value,
class Source, class Source,
class InnerValue = typename ValueTypeOfRange<Range>::RefType> class Result = typename std::remove_reference<Value>::type*>
class Generator class Generator : public GenImpl<Result, Generator<Value, Source, Result>> {
: public GenImpl<InnerValue, Generator<Range, Source, InnerValue>> {
Source source_; Source source_;
static_assert(!std::is_rvalue_reference<Value>::value,
"Cannot use indirect on an rvalue");
public: public:
explicit Generator(Source source) explicit Generator(Source source) : source_(std::move(source)) {}
: source_(std::move(source)) {}
template<class Body> template <class Body>
void foreach(Body&& body) const { void foreach(Body&& body) const {
source_.foreach([&](Range range) { source_.foreach([&](Value value) {
for (auto& value : range) { return body(&value);
body(value); });
}
});
} }
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
return source_.apply([&](Range range) -> bool { return source_.apply([&](Value value) -> bool {
for (auto& value : range) { return handler(&value);
if (!handler(value)) { });
return false;
}
}
return true;
});
} }
// Just passes value though, length unaffected
static constexpr bool infinite = Source::infinite;
}; };
template<class Value, template <class Source,
class Source, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self())); return Gen(std::move(source.self()));
} }
template<class Value, template <class Source,
class Source, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self()); return Gen(source.self());
} }
}; };
/** /**
* GuardImpl - For handling exceptions from downstream computation. Requires the * Cycle - For repeating a sequence forever.
* type of exception to catch, and handler function to invoke in the event of
* the exception. Note that the handler may:
* 1) return true to continue processing the sequence
* 2) return false to end the sequence immediately
* 3) throw, to pass the exception to the next catch
* The handler must match the signature 'bool(Exception&, Value)'.
* *
* This type is used through the `guard` helper, like so: * This type is usually used through the 'cycle' static value, like:
* *
* auto indexes * auto tests
* = byLine(STDIN_FILENO) * = from(samples)
* | guard<std::runtime_error>([](std::runtime_error& e, * | cycle
* StringPiece sp) { * | take(100);
* LOG(ERROR) << sp << ": " << e.str();
* return true; // continue processing subsequent lines
* })
* | eachTo<int>()
* | as<vector>();
* *
* TODO(tjackson): Rename this back to Guard. * or in the finite case:
**/ *
template<class Exception, * auto thrice = g | cycle(3);
class ErrorHandler> */
class GuardImpl : public Operator<GuardImpl<Exception, ErrorHandler>> { template <bool forever>
ErrorHandler handler_; class Cycle : public Operator<Cycle<forever>> {
off_t limit_; // not used if forever == true
public: public:
explicit GuardImpl(ErrorHandler handler) : handler_(std::move(handler)) {} Cycle() = default;
explicit Cycle(off_t limit) : limit_(limit) {
static_assert(
!forever,
"Cycle limit consturctor should not be used when forever == true.");
}
template<class Value, template <class Value, class Source>
class Source>
class Generator : public GenImpl<Value, Generator<Value, Source>> { class Generator : public GenImpl<Value, Generator<Value, Source>> {
Source source_; Source source_;
ErrorHandler handler_; off_t limit_;
public:
explicit Generator(Source source, public:
ErrorHandler handler) explicit Generator(Source source, off_t limit)
: source_(std::move(source)), : source_(std::move(source)), limit_(limit) {}
handler_(std::move(handler)) {}
template<class Handler> template <class Handler>
bool apply(Handler&& handler) const { bool apply(Handler&& handler) const {
return source_.apply([&](Value value) -> bool { bool cont;
try { auto handler2 = [&](Value value) {
handler(std::forward<Value>(value)); cont = handler(std::forward<Value>(value));
return true; return cont;
} catch (Exception& e) { };
return handler_(e, std::forward<Value>(value)); // Becomes an infinte loop if forever == true
for (off_t count = 0; (forever || count != limit_); ++count) {
cont = false;
source_.apply(handler2);
if (!cont) {
return false;
} }
}); }
return true;
} }
static constexpr bool infinite = Source::infinite; // This is the hardest one to infer. If we are simply doing a finite cycle,
// then (gen | cycle(n)) is infinite if and only if gen is infinite.
// However, if we are doing an infinite cycle, (gen | cycle) is infinite
// unless gen is empty. However, we will always mark (gen | cycle) as
// infinite, because patterns such as (gen | cycle | count) can either take
// on exactly one value, or infinite loop.
static constexpr bool infinite = forever || Source::infinite;
}; };
template<class Value, template <class Source,
class Source, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const { Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()), handler_); return Gen(std::move(source.self()), limit_);
} }
template<class Value, template <class Source,
class Source, class Value,
class Gen = Generator<Value, Source>> class Gen = Generator<Value, Source>>
Gen compose(const GenImpl<Value, Source>& source) const { Gen compose(const GenImpl<Value, Source>& source) const {
return Gen(source.self(), handler_); return Gen(source.self(), limit_);
}
/**
* Convenience function for finite cycles used like:
*
* auto tripled = gen | cycle(3);
*/
Cycle<false> operator()(off_t limit) const { return Cycle<false>(limit); }
};
/*
******************************* Sinks *****************************************
*/
/**
* FoldLeft - Left-associative functional fold. For producing an aggregate value
* from a seed and a folder function. Useful for custom aggregators on a
* sequence.
*
* This type is primarily used through the 'foldl' helper method, like:
*
* double movingAverage = from(values)
* | foldl(0.0, [](double avg, double sample) {
* return sample * 0.1 + avg * 0.9;
* });
*/
template <class Seed, class Fold>
class FoldLeft : public Operator<FoldLeft<Seed, Fold>> {
Seed seed_;
Fold fold_;
public:
FoldLeft() = default;
FoldLeft(Seed seed, Fold fold)
: seed_(std::move(seed)), fold_(std::move(fold)) {}
template <class Source, class Value>
Seed compose(const GenImpl<Value, Source>& source) const {
static_assert(!Source::infinite, "Cannot foldl infinite source");
Seed accum = seed_;
source | [&](Value v) {
accum = fold_(std::move(accum), std::forward<Value>(v));
};
return accum;
}
};
/**
* First - For finding the first value in a sequence.
*
* This type is primarily used through the 'first' static value, like:
*
* int firstThreeDigitPrime = seq(100) | filter(isPrime) | first;
*/
class First : public Operator<First> {
public:
First() = default;
template <class Source,
class Value,
class StorageType = typename std::decay<Value>::type>
StorageType compose(const GenImpl<Value, Source>& source) const {
Optional<StorageType> accum;
source | [&](Value v) -> bool {
accum = std::forward<Value>(v);
return false;
};
if (!accum.hasValue()) {
throw EmptySequence();
}
return std::move(accum.value());
}
};
/**
* IsEmpty - a helper class for isEmpty and notEmpty
*
* Essentially returns 'result' if the source is empty. Note that this cannot be
* called on an infinite source, because then there is only one possible return
* value.
*
*
* Used primarily through 'isEmpty' and 'notEmpty' static values
*
* bool hasPrimes = g | filter(prime) | notEmpty;
* bool lacksEvens = g | filter(even) | isEmpty;
*
* Also used in the implementation of 'any' and 'all'
*/
template <bool emptyResult>
class IsEmpty : public Operator<IsEmpty<emptyResult>> {
public:
IsEmpty() = default;
template <class Source, class Value>
bool compose(const GenImpl<Value, Source>& source) const {
static_assert(!Source::infinite,
"Cannot call 'all', 'any', 'isEmpty', or 'notEmpty' on "
"infinite source. 'all' and 'isEmpty' will either return "
"false or hang. 'any' or 'notEmpty' will either return true "
"or hang.");
bool ans = emptyResult;
source | [&](Value v) -> bool {
ans = !emptyResult;
return false;
};
return ans;
}
};
/**
* Reduce - Functional reduce, for recursively combining values from a source
* using a reducer function until there is only one item left. Useful for
* combining values when an empty sequence doesn't make sense.
*
* This type is primarily used through the 'reduce' helper method, like:
*
* sring longest = from(names)
* | reduce([](string&& best, string& current) {
* return best.size() >= current.size() ? best : current;
* });
*/
template <class Reducer>
class Reduce : public Operator<Reduce<Reducer>> {
Reducer reducer_;
public:
Reduce() = default;
explicit Reduce(Reducer reducer) : reducer_(std::move(reducer)) {}
template <class Source,
class Value,
class StorageType = typename std::decay<Value>::type>
StorageType compose(const GenImpl<Value, Source>& source) const {
static_assert(!Source::infinite, "Cannot reduce infinite source");
Optional<StorageType> accum;
source | [&](Value v) {
if (accum.hasValue()) {
accum = reducer_(std::move(accum.value()), std::forward<Value>(v));
} else {
accum = std::forward<Value>(v);
}
};
if (!accum.hasValue()) {
throw EmptySequence();
}
return accum.value();
}
};
/**
* Count - for simply counting the items in a collection.
*
* This type is usually used through its singleton, 'count':
*
* auto shortPrimes = seq(1, 100) | filter(isPrime) | count;
*/
class Count : public Operator<Count> {
public:
Count() = default;
template <class Source, class Value>
size_t compose(const GenImpl<Value, Source>& source) const {
static_assert(!Source::infinite, "Cannot count infinite source");
return foldl(size_t(0),
[](size_t accum, Value v) {
return accum + 1;
}).compose(source);
} }
}; };
/** /**
* Cycle - For repeating a sequence forever. * Sum - For simply summing up all the values from a source.
*
* This type is usually used through the 'cycle' static value, like:
*
* auto tests
* = from(samples)
* | cycle
* | take(100);
* *
* or in the finite case: * This type is usually used through its singleton, 'sum':
* *
* auto thrice = g | cycle(3); * auto gaussSum = seq(1, 100) | sum;
*/ */
template <bool forever> class Sum : public Operator<Sum> {
class Cycle : public Operator<Cycle<forever>> {
off_t limit_; // not used if forever == true
public: public:
Cycle() = default; Sum() = default;
explicit Cycle(off_t limit)
: limit_(limit) {
static_assert(
!forever,
"Cycle limit consturctor should not be used when forever == true.");
}
template<class Value,
class Source>
class Generator : public GenImpl<Value, Generator<Value, Source>> {
Source source_;
off_t limit_;
public:
explicit Generator(Source source, off_t limit)
: source_(std::move(source))
, limit_(limit) {}
template<class Handler>
bool apply(Handler&& handler) const {
bool cont;
auto handler2 = [&](Value value) {
cont = handler(std::forward<Value>(value));
return cont;
};
// Becomes an infinte loop if forever == true
for (off_t count = 0; (forever || count != limit_); ++count) {
cont = false;
source_.apply(handler2);
if (!cont) {
return false;
}
}
return true;
}
// not actually infinite, since an empty generator will end the cycles.
static constexpr bool infinite = Source::infinite;
};
template<class Source, template <class Source,
class Value, class Value,
class Gen = Generator<Value, Source>> class StorageType = typename std::decay<Value>::type>
Gen compose(GenImpl<Value, Source>&& source) const { StorageType compose(const GenImpl<Value, Source>& source) const {
return Gen(std::move(source.self()), limit_); static_assert(!Source::infinite, "Cannot sum infinite source");
return foldl(StorageType(0),
[](StorageType&& accum, Value v) {
return std::move(accum) + std::forward<Value>(v);
}).compose(source);
} }
};
template<class Source, /**
class Value, * Contains - For testing whether a value matching the given value is contained
class Gen = Generator<Value, Source>> * in a sequence.
Gen compose(const GenImpl<Value, Source>& source) const { *
return Gen(source.self(), limit_); * This type should be used through the 'contains' helper method, like:
} *
* bool contained = seq(1, 10) | map(square) | contains(49);
*/
template <class Needle>
class Contains : public Operator<Contains<Needle>> {
Needle needle_;
/** public:
* Convenience function for finite cycles used like: explicit Contains(Needle needle) : needle_(std::move(needle)) {}
*
* auto tripled = gen | cycle(3); template <class Source,
*/ class Value,
Cycle<false> operator()(off_t limit) const { class StorageType = typename std::decay<Value>::type>
return Cycle<false>(limit); bool compose(const GenImpl<Value, Source>& source) const {
static_assert(!Source::infinite,
"Calling contains on an infinite source might cause "
"an infinite loop.");
return !(source | [this](Value value) {
return !(needle_ == std::forward<Value>(value));
});
} }
}; };
/** /**
* Dereference - For dereferencing a sequence of pointers while filtering out * Min - For a value which minimizes a key, where the key is determined by a
* null pointers. * given selector, and compared by given comparer.
* *
* This type is usually used through the 'dereference' static value, like: * This type is usually used through the singletone 'min' or through the helper
* functions 'minBy' and 'maxBy'.
* *
* auto refs = from(ptrs) | dereference; * auto oldest = from(people)
* | minBy([](Person& p) {
* return p.dateOfBirth;
* });
*/ */
class Dereference : public Operator<Dereference> { template <class Selector, class Comparer>
class Min : public Operator<Min<Selector, Comparer>> {
Selector selector_;
Comparer comparer_;
public: public:
Dereference() = default; Min() = default;
template<class Value, explicit Min(Selector selector) : selector_(std::move(selector)) {}
class Source,
class Result = decltype(*std::declval<Value>())>
class Generator : public GenImpl<Result, Generator<Value, Source, Result>> {
Source source_;
public:
explicit Generator(Source source)
: source_(std::move(source)) {}
template<class Body> Min(Selector selector, Comparer comparer)
void foreach(Body&& body) const { : selector_(std::move(selector)), comparer_(std::move(comparer)) {}
source_.foreach([&](Value value) {
if (value) {
return body(*value);
}
});
}
template<class Handler> template <class Value,
bool apply(Handler&& handler) const { class Source,
return source_.apply([&](Value value) -> bool { class StorageType = typename std::decay<Value>::type,
if (value) { class Key = typename std::decay<
return handler(*value); typename std::result_of<Selector(Value)>::type>::type>
} StorageType compose(const GenImpl<Value, Source>& source) const {
return true; static_assert(!Source::infinite,
}); "Calling min or max on an infinite source will cause "
"an infinite loop.");
Optional<StorageType> min;
Optional<Key> minKey;
source | [&](Value v) {
Key key = selector_(std::forward<Value>(v));
if (!minKey.hasValue() || comparer_(key, minKey.value())) {
minKey = key;
min = std::forward<Value>(v);
}
};
if (!min.hasValue()) {
throw EmptySequence();
} }
return min.value();
}
};
// not actually infinite, since an empty generator will end the cycles. /**
static constexpr bool infinite = Source::infinite; * Append - For collecting values from a source into a given output container
}; * by appending.
*
* This type is usually used through the helper function 'appendTo', like:
*
* vector<int64_t> ids;
* from(results) | map([](Person& p) { return p.id })
* | appendTo(ids);
*/
template <class Collection>
class Append : public Operator<Append<Collection>> {
Collection* collection_;
template<class Source, public:
class Value, explicit Append(Collection* collection) : collection_(collection) {}
class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()));
}
template<class Source, template <class Value, class Source>
class Value, Collection& compose(const GenImpl<Value, Source>& source) const {
class Gen = Generator<Value, Source>> static_assert(!Source::infinite, "Cannot appendTo with infinite source");
Gen compose(const GenImpl<Value, Source>& source) const { source | [&](Value v) {
return Gen(source.self()); collection_->insert(collection_->end(), std::forward<Value>(v));
};
return *collection_;
} }
}; };
/** /**
* Indirect - For producing a sequence of the addresses of the values in the * Collect - For collecting values from a source in a collection of the desired
* input. * type.
* *
* This type is usually used through the 'indirect' static value, like: * This type is usually used through the helper function 'as', like:
* *
* auto ptrs = from(refs) | indirect; * std::string upper = from(stringPiece)
* | map(&toupper)
* | as<std::string>();
*/ */
class Indirect : public Operator<Indirect> { template <class Collection>
class Collect : public Operator<Collect<Collection>> {
public: public:
Indirect() = default; Collect() = default;
template <class Value, template <class Value,
class Source, class Source,
class Result = typename std::remove_reference<Value>::type*> class StorageType = typename std::decay<Value>::type>
class Generator : public GenImpl<Result, Generator<Value, Source, Result>> { Collection compose(const GenImpl<Value, Source>& source) const {
Source source_; static_assert(!Source::infinite,
static_assert(!std::is_rvalue_reference<Value>::value, "Cannot convert infinite source to object with as.");
"Cannot use indirect on an rvalue"); Collection collection;
source | [&](Value v) {
public: collection.insert(collection.end(), std::forward<Value>(v));
explicit Generator(Source source) : source_(std::move(source)) {} };
return collection;
template <class Body>
void foreach(Body&& body) const {
source_.foreach([&](Value value) {
return body(&value);
});
}
template <class Handler>
bool apply(Handler&& handler) const {
return source_.apply([&](Value value) -> bool {
return handler(&value);
});
}
// not actually infinite, since an empty generator will end the cycles.
static constexpr bool infinite = Source::infinite;
};
template <class Source, class Value, class Gen = Generator<Value, Source>>
Gen compose(GenImpl<Value, Source>&& source) const {
return Gen(std::move(source.self()));
} }
};
template <class Source, class Value, class Gen = Generator<Value, Source>> /**
Gen compose(const GenImpl<Value, Source>& source) const { * CollectTemplate - For collecting values from a source in a collection
return Gen(source.self()); * constructed using the specified template type. Given the type of values
* produced by the given generator, the collection type will be:
* Container<Value, Allocator<Value>>
*
* The allocator defaults to std::allocator, so this may be used for the STL
* containers by simply using operators like 'as<set>', 'as<deque>',
* 'as<vector>'. 'as', here is the helper method which is the usual means of
* consturcting this operator.
*
* Example:
*
* set<string> uniqueNames = from(names) | as<set>();
*/
template <template <class, class> class Container,
template <class> class Allocator>
class CollectTemplate : public Operator<CollectTemplate<Container, Allocator>> {
public:
CollectTemplate() = default;
template <class Value,
class Source,
class StorageType = typename std::decay<Value>::type,
class Collection = Container<StorageType, Allocator<StorageType>>>
Collection compose(const GenImpl<Value, Source>& source) const {
static_assert(!Source::infinite,
"Cannot convert infinite source to object with as.");
Collection collection;
source | [&](Value v) {
collection.insert(collection.end(), std::forward<Value>(v));
};
return collection;
} }
}; };
...@@ -2022,7 +2098,7 @@ class Indirect : public Operator<Indirect> { ...@@ -2022,7 +2098,7 @@ class Indirect : public Operator<Indirect> {
/** /**
* VirtualGen<T> - For wrapping template types in simple polymorphic wrapper. * VirtualGen<T> - For wrapping template types in simple polymorphic wrapper.
**/ **/
template<class Value> template <class Value>
class VirtualGen : public GenImpl<Value, VirtualGen<Value>> { class VirtualGen : public GenImpl<Value, VirtualGen<Value>> {
class WrapperBase { class WrapperBase {
public: public:
...@@ -2032,13 +2108,12 @@ class VirtualGen : public GenImpl<Value, VirtualGen<Value>> { ...@@ -2032,13 +2108,12 @@ class VirtualGen : public GenImpl<Value, VirtualGen<Value>> {
virtual std::unique_ptr<const WrapperBase> clone() const = 0; virtual std::unique_ptr<const WrapperBase> clone() const = 0;
}; };
template<class Wrapped> template <class Wrapped>
class WrapperImpl : public WrapperBase { class WrapperImpl : public WrapperBase {
Wrapped wrapped_; Wrapped wrapped_;
public: public:
explicit WrapperImpl(Wrapped wrapped) explicit WrapperImpl(Wrapped wrapped) : wrapped_(std::move(wrapped)) {}
: wrapped_(std::move(wrapped)) {
}
virtual bool apply(const std::function<bool(Value)>& handler) const { virtual bool apply(const std::function<bool(Value)>& handler) const {
return wrapped_.apply(handler); return wrapped_.apply(handler);
...@@ -2063,8 +2138,7 @@ class VirtualGen : public GenImpl<Value, VirtualGen<Value>> { ...@@ -2063,8 +2138,7 @@ class VirtualGen : public GenImpl<Value, VirtualGen<Value>> {
VirtualGen(VirtualGen&& source) noexcept VirtualGen(VirtualGen&& source) noexcept
: wrapper_(std::move(source.wrapper_)) {} : wrapper_(std::move(source.wrapper_)) {}
VirtualGen(const VirtualGen& source) VirtualGen(const VirtualGen& source) : wrapper_(source.wrapper_->clone()) {}
: wrapper_(source.wrapper_->clone()) {}
VirtualGen& operator=(const VirtualGen& source) { VirtualGen& operator=(const VirtualGen& source) {
wrapper_.reset(source.wrapper_->clone()); wrapper_.reset(source.wrapper_->clone());
...@@ -2072,7 +2146,7 @@ class VirtualGen : public GenImpl<Value, VirtualGen<Value>> { ...@@ -2072,7 +2146,7 @@ class VirtualGen : public GenImpl<Value, VirtualGen<Value>> {
} }
VirtualGen& operator=(VirtualGen&& source) noexcept { VirtualGen& operator=(VirtualGen&& source) noexcept {
wrapper_= std::move(source.wrapper_); wrapper_ = std::move(source.wrapper_);
return *this; return *this;
} }
...@@ -2095,12 +2169,6 @@ constexpr detail::Count count{}; ...@@ -2095,12 +2169,6 @@ constexpr detail::Count count{};
constexpr detail::First first{}; constexpr detail::First first{};
/**
* Use 'isEmpty' and 'notEmpty' for detecting if there are any values or not.
*
* bool hasPrimes = g | filter(prime) | notEmpty;
* bool lacksEvens = g | filter(even) | isEmpty;
*/
constexpr detail::IsEmpty<true> isEmpty{}; constexpr detail::IsEmpty<true> isEmpty{};
constexpr detail::IsEmpty<false> notEmpty{}; constexpr detail::IsEmpty<false> notEmpty{};
...@@ -2125,27 +2193,21 @@ constexpr detail::Dereference dereference{}; ...@@ -2125,27 +2193,21 @@ constexpr detail::Dereference dereference{};
constexpr detail::Indirect indirect{}; constexpr detail::Indirect indirect{};
inline detail::Take take(size_t count) { inline detail::Take take(size_t count) { return detail::Take(count); }
return detail::Take(count);
}
inline detail::Stride stride(size_t s) { inline detail::Stride stride(size_t s) { return detail::Stride(s); }
return detail::Stride(s);
}
template<class Random = std::default_random_engine> template <class Random = std::default_random_engine>
inline detail::Sample<Random> sample(size_t count, Random rng = Random()) { inline detail::Sample<Random> sample(size_t count, Random rng = Random()) {
return detail::Sample<Random>(count, std::move(rng)); return detail::Sample<Random>(count, std::move(rng));
} }
inline detail::Skip skip(size_t count) { inline detail::Skip skip(size_t count) { return detail::Skip(count); }
return detail::Skip(count);
}
inline detail::Batch batch(size_t batchSize) { inline detail::Batch batch(size_t batchSize) {
return detail::Batch(batchSize); return detail::Batch(batchSize);
} }
}} //folly::gen }} // folly::gen
#pragma GCC diagnostic pop #pragma GCC diagnostic pop
folly/gen/Core-inl.h
View file @ a1107271
...@@ -181,6 +181,13 @@ class GenImpl : public FBounded<Self> { ...@@ -181,6 +181,13 @@ class GenImpl : public FBounded<Self> {
// Child classes should override if the sequence generated is *definitely* // Child classes should override if the sequence generated is *definitely*
// infinite. 'infinite' may be false_type for some infinite sequences // infinite. 'infinite' may be false_type for some infinite sequences
// (due the the Halting Problem). // (due the the Halting Problem).
//
// In general, almost all sources are finite (only seq(n) produces an infinite
// source), almost all operators keep the finiteness of the source (only cycle
// makes an infinite generator from a finite one, only until and take make a
// finite generator from an infinite one, and concat needs both the inner and
// outer generators to be finite to make a finite one), and most sinks
// cannot accept and infinite generators (first being the expection).
static constexpr bool infinite = false; static constexpr bool infinite = false;
}; };
......
folly/gen/test/BaseTest.cpp
View file @ a1107271
...@@ -577,6 +577,37 @@ TEST(Gen, DistinctMove) { // 0 1 4 9 6 5 6 9 4 1 0 ...@@ -577,6 +577,37 @@ TEST(Gen, DistinctMove) { // 0 1 4 9 6 5 6 9 4 1 0
EXPECT_EQ(expected, actual); EXPECT_EQ(expected, actual);
} }
TEST(Gen, DistinctInfinite) {
// distinct should be able to handle an infinite sequence, provided that, of
// of cource, is it eventually made finite before returning the result.
auto expected = seq(0) | take(5) | as<vector>(); // 0 1 2 3 4
auto actual =
seq(0) // 0 1 2 3 4 5 6 7 ...
| mapped([](int i) { return i / 2; }) // 0 0 1 1 2 2 3 3 ...
| distinct // 0 1 2 3 4 5 6 7 ...
| take(5) // 0 1 2 3 4
| as<vector>();
EXPECT_EQ(expected, actual);
}
TEST(Gen, DistinctByInfinite) {
// Similarly to the DistinctInfinite test case, distinct by should be able to
// handle infinite sequences. Note that depending on how many values we take()
// at the end, the sequence may infinite loop. This is fine becasue we cannot
// solve the halting problem.
auto expected = vector<int>{1, 2};
auto actual =
seq(1) // 1 2 3 4 5 6 7 8 ...
| distinctBy([](int i) { return i % 2; }) // 1 2 (but might by infinite)
| take(2) // 1 2
| as<vector>();
// Note that if we had take(3), this would infinite loop
EXPECT_EQ(expected, actual);
}
TEST(Gen, MinBy) { TEST(Gen, MinBy) {
EXPECT_EQ(7, seq(1, 10) EXPECT_EQ(7, seq(1, 10)
| minBy([](int i) -> double { | minBy([](int i) -> double {
...@@ -743,21 +774,17 @@ TEST(Gen, Get) { ...@@ -743,21 +774,17 @@ TEST(Gen, Get) {
} }
TEST(Gen, notEmpty) { TEST(Gen, notEmpty) {
EXPECT_TRUE(seq(0) | notEmpty);
EXPECT_TRUE(seq(0, 1) | notEmpty); EXPECT_TRUE(seq(0, 1) | notEmpty);
EXPECT_TRUE(just(1) | notEmpty); EXPECT_TRUE(just(1) | notEmpty);
EXPECT_FALSE(gen::range(0, 0) | notEmpty); EXPECT_FALSE(gen::range(0, 0) | notEmpty);
EXPECT_FALSE(from({1}) | take(0) | notEmpty); EXPECT_FALSE(from({1}) | take(0) | notEmpty);
EXPECT_TRUE(seq(1, 3) | cycle | notEmpty);
} }
TEST(Gen, isEmpty) { TEST(Gen, isEmpty) {
EXPECT_FALSE(seq(0) | isEmpty);
EXPECT_FALSE(seq(0, 1) | isEmpty); EXPECT_FALSE(seq(0, 1) | isEmpty);
EXPECT_FALSE(just(1) | isEmpty); EXPECT_FALSE(just(1) | isEmpty);
EXPECT_TRUE(gen::range(0, 0) | isEmpty); EXPECT_TRUE(gen::range(0, 0) | isEmpty);
EXPECT_TRUE(from({1}) | take(0) | isEmpty); EXPECT_TRUE(from({1}) | take(0) | isEmpty);
EXPECT_FALSE(seq(1, 3) | cycle | isEmpty);
} }
TEST(Gen, Any) { TEST(Gen, Any) {
...@@ -1018,7 +1045,8 @@ TEST(Gen, Cycle) { ...@@ -1018,7 +1045,8 @@ TEST(Gen, Cycle) {
}; };
auto s = countdown; auto s = countdown;
EXPECT_EQ((vector<int> { 1, 2, 3, 1, 2, 1}), EXPECT_EQ((vector<int> { 1, 2, 3, 1, 2, 1}),
s | cycle | as<vector>()); s | cycle | take(7) | as<vector>());
// take necessary as cycle returns an infinite generator
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment