Use seq to break a tie for stream weight

Because of the nature of heap based priority queue, and our compare
function, streams with the same weight and same parent are handled in
the reverse order they pushed to the queue.  To allow stream pushed
earlier to be served first, secondary key "seq" is introduced to break
a tie.  "seq" is monotonically increased integer per parent stream,
and it is assigned to stream when it is pused to the queue, and gets
incremented.

lib/nghttp2_stream.c

@@ -30,13 +30,14 @@
 #include "nghttp2_session.h"
 #include "nghttp2_helper.h"
 
-static int stream_weight_less(const void *lhsx, const void *rhsx) {
+static int stream_less(const void *lhsx, const void *rhsx) {
   const nghttp2_stream *lhs, *rhs;
 
   lhs = nghttp2_struct_of(lhsx, nghttp2_stream, pq_entry);
   rhs = nghttp2_struct_of(rhsx, nghttp2_stream, pq_entry);
 
-  return lhs->cycle < rhs->cycle;
+  return lhs->cycle < rhs->cycle ||
+         (lhs->cycle == rhs->cycle && lhs->seq < rhs->seq);
 }
 
 void nghttp2_stream_init(nghttp2_stream *stream, int32_t stream_id,
@@ -45,7 +46,7 @@ void nghttp2_stream_init(nghttp2_stream *stream, int32_t stream_id,
                          int32_t local_initial_window_size,
                          void *stream_user_data, nghttp2_mem *mem) {
   nghttp2_map_entry_init(&stream->map_entry, (key_type)stream_id);
-  nghttp2_pq_init(&stream->obq, stream_weight_less, mem);
+  nghttp2_pq_init(&stream->obq, stream_less, mem);
 
   stream->stream_id = stream_id;
   stream->flags = flags;
@@ -79,6 +80,8 @@ void nghttp2_stream_init(nghttp2_stream *stream, int32_t stream_id,
   stream->queued = 0;
   stream->descendant_last_cycle = 0;
   stream->cycle = 0;
+  stream->descendant_next_seq = 0;
+  stream->seq = 0;
   stream->last_writelen = 0;
 }
 
@@ -124,6 +127,7 @@ static int stream_obq_push(nghttp2_stream *dep_stream, nghttp2_stream *stream) {
        stream = dep_stream, dep_stream = dep_stream->dep_prev) {
     stream->cycle =
         stream_next_cycle(stream, dep_stream->descendant_last_cycle);
+    stream->seq = dep_stream->descendant_next_seq++;
 
     DEBUGF(fprintf(stderr, "stream: stream=%d obq push cycle=%ld\n",
                    stream->stream_id, stream->cycle));
@@ -214,10 +218,12 @@ void nghttp2_stream_reschedule(nghttp2_stream *stream) {
          just makes new streams scheduled a bit early. */
       dep_stream->descendant_last_cycle = stream->cycle;
 
+      nghttp2_pq_remove(&dep_stream->obq, &stream->pq_entry);
+
       stream->cycle =
           stream_next_cycle(stream, dep_stream->descendant_last_cycle);
+      stream->seq = dep_stream->descendant_next_seq++;
 
-      nghttp2_pq_remove(&dep_stream->obq, &stream->pq_entry);
       nghttp2_pq_push(&dep_stream->obq, &stream->pq_entry);
     }
 

lib/nghttp2_stream.h

@@ -206,6 +206,11 @@ struct nghttp2_stream {
   uint64_t descendant_last_cycle;
   /* Next scheduled time to sent item */
   uint64_t cycle;
+  /* Next seq used for direct descendant streams */
+  uint64_t descendant_next_seq;
+  /* Secondary key for prioritization to break a tie for cycle.  This
+     value is monotonically increased for single parent stream. */
+  uint64_t seq;
   /* Last written length of frame payload */
   size_t last_writelen;
   /* This flag is used to reduce excessive queuing of WINDOW_UPDATE to