gstreamer插件编写教程2-Advanced Concepts

索引：https://blog.csdn.net/knowledgebao/article/details/84621238

文档教程：https://gstreamer.freedesktop.org/documentation/plugin-development/advanced/index.html

---

 

目录

Advanced Concepts

Request and Sometimes pads

Different scheduling modes

Caps negotiation

Memory allocation

Media Types and Properties

Events: Seeking, Navigation and More

Clocking

Quality Of Service (QoS)

Supporting Dynamic Parameters

Interfaces

Tagging (Metadata and Streaminfo)

---

 

Advanced Concepts

到目前为止，您应该能够创建可以接收和发送数据的基本过滤器元素。这是GStreamer所代表的简单模型。但是GStreamer可以做的不仅仅是这个！在本章中，将讨论各种高级主题，例如调度，特殊键盘类型，时钟，事件，接口，标记等。这些主题是使GStreamer易用于应用程序的糖果。

By now, you should be able to create basic filter elements that can receive and send data. This is the simple model that GStreamer stands for. But GStreamer can do much more than only this! In this chapter, various advanced topics will be discussed, such as scheduling, special pad types, clocking, events, interfaces, tagging and more. These topics are the sugar that makes GStreamer so easy to use for applications.

Request and Sometimes pads

Until now, we've only dealt with pads that are always available. However, there's also pads that are only being created in some cases, or only if the application requests the pad. The first is called a sometimes; the second is called a request pad. The availability of a pad (always, sometimes or request) can be seen in a pad's template. This chapter will discuss when each of the two is useful, how they are created and when they should be disposed.

Sometimes pads

A “sometimes” pad is a pad that is created under certain conditions, but not in all cases. This mostly depends on stream content: demuxers will generally parse the stream header, decide what elementary (video, audio, subtitle, etc.) streams are embedded inside the system stream, and will then create a sometimes pad for each of those elementary streams. At its own choice, it can also create more than one instance of each of those per element instance. The only limitation is that each newly created pad should have a unique name. Sometimes pads are disposed when the stream data is disposed, too (i.e. when going from PAUSED to the READY state). You should not dispose the pad on EOS, because someone might re-activate the pipeline and seek back to before the end-of-stream point. The stream should still stay valid after EOS, at least until the stream data is disposed. In any case, the element is always the owner of such a pad.

The example code below will parse a text file, where the first line is a number (n). The next lines all start with a number (0 to n-1), which is the number of the source pad over which the data should be sent.

3
0: foo
1: bar
0: boo
2: bye

The code to parse this file and create the dynamic “sometimes” pads, looks like this:

typedef struct _GstMyFilter {
[..]
  gboolean firstrun;
  GList *srcpadlist;
} GstMyFilter;

static GstStaticPadTemplate src_factory =
GST_STATIC_PAD_TEMPLATE (
  "src_%u",
  GST_PAD_SRC,
  GST_PAD_SOMETIMES,
  GST_STATIC_CAPS ("ANY")
);

static void
gst_my_filter_class_init (GstMyFilterClass *klass)
{
  GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
[..]
  gst_element_class_add_pad_template (element_class,
    gst_static_pad_template_get (&src_factory));
[..]
}

static void
gst_my_filter_init (GstMyFilter *filter)
{
[..]
  filter->firstrun = TRUE;
  filter->srcpadlist = NULL;
}

/*
 * Get one line of data - without newline.
 */

static GstBuffer *
gst_my_filter_getline (GstMyFilter *filter)
{
  guint8 *data;
  gint n, num;

  /* max. line length is 512 characters - for safety */
  for (n = 0; n < 512; n++) {
    num = gst_bytestream_peek_bytes (filter->bs, &data, n + 1);
    if (num != n + 1)
      return NULL;

    /* newline? */
    if (data[n] == '\n') {
      GstBuffer *buf = gst_buffer_new_allocate (NULL, n + 1, NULL);

      gst_bytestream_peek_bytes (filter->bs, &data, n);
      gst_buffer_fill (buf, 0, data, n);
      gst_buffer_memset (buf, n, '\0', 1);
      gst_bytestream_flush_fast (filter->bs, n + 1);

      return buf;
    }
  }
}

static void
gst_my_filter_loopfunc (GstElement *element)
{
  GstMyFilter *filter = GST_MY_FILTER (element);
  GstBuffer *buf;
  GstPad *pad;
  GstMapInfo map;
  gint num, n;

  /* parse header */
  if (filter->firstrun) {
    gchar *padname;
    guint8 id;

    if (!(buf = gst_my_filter_getline (filter))) {
      gst_element_error (element, STREAM, READ, (NULL),
             ("Stream contains no header"));
      return;
    }
    gst_buffer_extract (buf, 0, &id, 1);
    num = atoi (id);
    gst_buffer_unref (buf);

    /* for each of the streams, create a pad */
    for (n = 0; n < num; n++) {
      padname = g_strdup_printf ("src_%u", n);
      pad = gst_pad_new_from_static_template (src_factory, padname);
      g_free (padname);

      /* here, you would set _event () and _query () functions */

      /* need to activate the pad before adding */
      gst_pad_set_active (pad, TRUE);

      gst_element_add_pad (element, pad);
      filter->srcpadlist = g_list_append (filter->srcpadlist, pad);
    }
  }

  /* and now, simply parse each line and push over */
  if (!(buf = gst_my_filter_getline (filter))) {
    GstEvent *event = gst_event_new (GST_EVENT_EOS);
    GList *padlist;

    for (padlist = srcpadlist;
         padlist != NULL; padlist = g_list_next (padlist)) {
      pad = GST_PAD (padlist->data);
      gst_pad_push_event (pad, gst_event_ref (event));
    }
    gst_event_unref (event);
    /* pause the task here */
    return;
  }

  /* parse stream number and go beyond the ':' in the data */
  gst_buffer_map (buf, &map, GST_MAP_READ);
  num = atoi (map.data[0]);
  if (num >= 0 && num < g_list_length (filter->srcpadlist)) {
    pad = GST_PAD (g_list_nth_data (filter->srcpadlist, num);

    /* magic buffer parsing foo */
    for (n = 0; map.data[n] != ':' &&
                map.data[n] != '\0'; n++) ;
    if (map.data[n] != '\0') {
      GstBuffer *sub;

      /* create region copy that starts right past the space. The reason
       * that we don't just forward the data pointer is because the
       * pointer is no longer the start of an allocated block of memory,
       * but just a pointer to a position somewhere in the middle of it.
       * That cannot be freed upon disposal, so we'd either crash or have
       * a memleak. Creating a region copy is a simple way to solve that. */
      sub = gst_buffer_copy_region (buf, GST_BUFFER_COPY_ALL,
          n + 1, map.size - n - 1);
      gst_pad_push (pad, sub);
    }
  }
  gst_buffer_unmap (buf, &map);
  gst_buffer_unref (buf);
}

Note that we use a lot of checks everywhere to make sure that the content in the file is valid. This has two purposes: first, the file could be erroneous, in which case we prevent a crash. The second and most important reason is that - in extreme cases - the file could be used maliciously to cause undefined behaviour in the plugin, which might lead to security issues. Always assume that the file could be used to do bad things.

Request pads

“Request” pads are similar to sometimes pads, except that request are created on demand of something outside of the element rather than something inside the element. This concept is often used in muxers, where - for each elementary stream that is to be placed in the output system stream - one sink pad will be requested. It can also be used in elements with a variable number of input or outputs pads, such as the tee (multi-output) or input-selector (multi-input) elements.

To implement request pads, you need to provide a padtemplate with a GST_PAD_REQUEST presence and implement the request_new_pad virtual method in GstElement. To clean up, you will need to implement therelease_pad virtual method.

static GstPad * gst_my_filter_request_new_pad   (GstElement     *element,
                         GstPadTemplate *templ,
                                                 const gchar    *name,
                                                 const GstCaps  *caps);

static void gst_my_filter_release_pad (GstElement *element,
                                       GstPad *pad);

static GstStaticPadTemplate sink_factory =
GST_STATIC_PAD_TEMPLATE (
  "sink_%u",
  GST_PAD_SINK,
  GST_PAD_REQUEST,
  GST_STATIC_CAPS ("ANY")
);

static void
gst_my_filter_class_init (GstMyFilterClass *klass)
{
  GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
[..]
  gst_element_class_add_pad_template (klass,
    gst_static_pad_template_get (&sink_factory));
[..]
  element_class->request_new_pad = gst_my_filter_request_new_pad;
  element_class->release_pad = gst_my_filter_release_pad;
}

static GstPad *
gst_my_filter_request_new_pad (GstElement     *element,
                   GstPadTemplate *templ,
                   const gchar    *name,
                               const GstCaps  *caps)
{
  GstPad *pad;
  GstMyFilterInputContext *context;

  context = g_new0 (GstMyFilterInputContext, 1);
  pad = gst_pad_new_from_template (templ, name);
  gst_pad_set_element_private (pad, context);

  /* normally, you would set _chain () and _event () functions here */

  gst_element_add_pad (element, pad);

  return pad;
}

static void
gst_my_filter_release_pad (GstElement *element,
                           GstPad *pad)
{
  GstMyFilterInputContext *context;

  context = gst_pad_get_element_private (pad);
  g_free (context);

  gst_element_remove_pad (element, pad);
}

Different scheduling modes

The scheduling mode of a pad defines how data is retrieved from (source) or given to (sink) pads. GStreamer can operate in two scheduling mode, called push- and pull-mode. GStreamer supports elements with pads in any of the scheduling modes where not all pads need to be operating in the same mode.

So far, we have only discussed _chain ()-operating elements, i.e. elements that have a chain-function set on their sink pad and push buffers on their source pad(s). We call this the push-mode because a peer element will use gst_pad_push () on a srcpad, which will cause our _chain ()-function to be called, which in turn causes our element to push out a buffer on the source pad. The initiative to start the dataflow happens somewhere upstream when it pushes out a buffer and all downstream elements get scheduled when their _chain ()-functions are called in turn.

Before we explain pull-mode scheduling, let's first understand how the different scheduling modes are selected and activated on a pad.

The pad activation stage

During the element state change of READY->PAUSED, the pads of an element will be activated. This happens first on the source pads and then on the sink pads of the element. GStreamer calls the _activate () of a pad. By default this function will activate the pad in push-mode by calling gst_pad_activate_mode () with the GST_PAD_MODE_PUSH scheduling mode. It is possible to override the _activate () of a pad and decide on a different scheduling mode. You can know in what scheduling mode a pad is activated by overriding the _activate_mode ()-function.

GStreamer allows the different pads of an element to operate in different scheduling modes. This allows for many different possible use-cases. What follows is an overview of some typical use-cases.

• If all pads of an element are activated in push-mode scheduling, the element as a whole is operating in push-mode. For source elements this means that they will have to start a task that pushes out buffers on the source pad to the downstream elements. Downstream elements will have data pushed to them by upstream elements using the sinkpads _chain ()-function which will push out buffers on the source pads. Prerequisites for this scheduling mode are that a chain-function was set for each sinkpad usinggst_pad_set_chain_function () and that all downstream elements operate in the same mode.

• Alternatively, sinkpads can be the driving force behind a pipeline by operating in pull-mode, while the sourcepads of the element still operate in push-mode. In order to be the driving force, those pads start a GstTask when they are activated. This task is a thread, which will call a function specified by the element. When called, this function will have random data access (through gst_pad_pull_range ()) over all sinkpads, and can push data over the sourcepads, which effectively means that this element controls data flow in the pipeline. Prerequisites for this mode are that all downstream elements can act in push mode, and that all upstream elements operate in pull-mode (see below).

  Source pads can be activated in PULL mode by a downstream element when they return GST_PAD_MODE_PULL from the GST_QUERY_SCHEDULING query. Prerequisites for this scheduling mode are that a getrange-function was set for the source pad using gst_pad_set_getrange_function ().

• Lastly, all pads in an element can be activated in PULL-mode. However, contrary to the above, this does not mean that they start a task on their own. Rather, it means that they are pull slave for the downstream element, and have to provide random data access to it from their _get_range ()-function. Requirements are that the a _get_range ()-function was set on this pad using the function gst_pad_set_getrange_function (). Also, if the element has any sinkpads, all those pads (and thereby their peers) need to operate in PULL access mode, too.

  When a sink element is activated in PULL mode, it should start a task that calls gst_pad_pull_range () on its sinkpad. It can only do this when the upstream SCHEDULING query returns support for the GST_PAD_MODE_PULL scheduling mode.

In the next two sections, we will go closer into pull-mode scheduling (elements/pads driving the pipeline, and elements/pads providing random access), and some specific use cases will be given.

Pads driving the pipeline

Sinkpads operating in pull-mode, with the sourcepads operating in push-mode (or it has no sourcepads when it is a sink), can start a task that will drive the pipeline data flow. Within this task function, you have random access over all of the sinkpads, and push data over the sourcepads. This can come in useful for several different kinds of elements:

• Demuxers, parsers and certain kinds of decoders where data comes in unparsed (such as MPEG-audio or video streams), since those will prefer byte-exact (random) access from their input. If possible, however, such elements should be prepared to operate in push-mode mode, too.

• Certain kind of audio outputs, which require control over their input data flow, such as the Jack sound server.

First you need to perform a SCHEDULING query to check if the upstream element(s) support pull-mode scheduling. If that is possible, you can activate the sinkpad in pull-mode. Inside the activate_mode function you can then start the task.

#include "filter.h"
#include 

static gboolean gst_my_filter_activate      (GstPad      * pad,
                                             GstObject   * parent);
static gboolean gst_my_filter_activate_mode (GstPad      * pad,
                                             GstObject   * parent,
                                             GstPadMode    mode,
                         gboolean      active);
static void gst_my_filter_loop      (GstMyFilter * filter);

G_DEFINE_TYPE (GstMyFilter, gst_my_filter, GST_TYPE_ELEMENT);


static void
gst_my_filter_init (GstMyFilter * filter)
{

[..]

  gst_pad_set_activate_function (filter->sinkpad, gst_my_filter_activate);
  gst_pad_set_activatemode_function (filter->sinkpad,
      gst_my_filter_activate_mode);


[..]
}

[..]

static gboolean
gst_my_filter_activate (GstPad * pad, GstObject * parent)
{
  GstQuery *query;
  gboolean pull_mode;

  /* first check what upstream scheduling is supported */
  query = gst_query_new_scheduling ();

  if (!gst_pad_peer_query (pad, query)) {
    gst_query_unref (query);
    goto activate_push;
  }

  /* see if pull-mode is supported */
  pull_mode = gst_query_has_scheduling_mode_with_flags (query,
      GST_PAD_MODE_PULL, GST_SCHEDULING_FLAG_SEEKABLE);
  gst_query_unref (query);

  if (!pull_mode)
    goto activate_push;

  /* now we can activate in pull-mode. GStreamer will also
   * activate the upstream peer in pull-mode */
  return gst_pad_activate_mode (pad, GST_PAD_MODE_PULL, TRUE);

activate_push:
  {
    /* something not right, we fallback to push-mode */
    return gst_pad_activate_mode (pad, GST_PAD_MODE_PUSH, TRUE);
  }
}

static gboolean
gst_my_filter_activate_pull (GstPad    * pad,
                 GstObject * parent,
                 GstPadMode  mode,
                 gboolean    active)
{
  gboolean res;
  GstMyFilter *filter = GST_MY_FILTER (parent);

  switch (mode) {
    case GST_PAD_MODE_PUSH:
      res = TRUE;
      break;
    case GST_PAD_MODE_PULL:
      if (active) {
        filter->offset = 0;
        res = gst_pad_start_task (pad,
            (GstTaskFunction) gst_my_filter_loop, filter, NULL);
      } else {
        res = gst_pad_stop_task (pad);
      }
      break;
    default:
      /* unknown scheduling mode */
      res = FALSE;
      break;
  }
  return res;
}

Once started, your task has full control over input and output. The most simple case of a task function is one that reads input and pushes that over its source pad. It's not all that useful, but provides some more flexibility than the old push-mode case that we've been looking at so far.

    #define BLOCKSIZE 2048

    static void
    gst_my_filter_loop (GstMyFilter * filter)
    {
      GstFlowReturn ret;
      guint64 len;
      GstFormat fmt = GST_FORMAT_BYTES;
      GstBuffer *buf = NULL;

      if (!gst_pad_query_duration (filter->sinkpad, fmt, &len)) {
        GST_DEBUG_OBJECT (filter, "failed to query duration, pausing");
        goto stop;
      }

       if (filter->offset >= len) {
        GST_DEBUG_OBJECT (filter, "at end of input, sending EOS, pausing");
        gst_pad_push_event (filter->srcpad, gst_event_new_eos ());
        goto stop;
      }

      /* now, read BLOCKSIZE bytes from byte offset filter->offset */
      ret = gst_pad_pull_range (filter->sinkpad, filter->offset,
          BLOCKSIZE, &buf);

      if (ret != GST_FLOW_OK) {
        GST_DEBUG_OBJECT (filter, "pull_range failed: %s", gst_flow_get_name (ret));
        goto stop;
      }

      /* now push buffer downstream */
      ret = gst_pad_push (filter->srcpad, buf);

      buf = NULL; /* gst_pad_push() took ownership of buffer */

      if (ret != GST_FLOW_OK) {
        GST_DEBUG_OBJECT (filter, "pad_push failed: %s", gst_flow_get_name (ret));
        goto stop;
      }

      /* everything is fine, increase offset and wait for us to be called again */
      filter->offset += BLOCKSIZE;
      return;

    stop:
      GST_DEBUG_OBJECT (filter, "pausing task");
      gst_pad_pause_task (filter->sinkpad);
    }

Providing random access

In the previous section, we have talked about how elements (or pads) that are activated to drive the pipeline using their own task, must use pull-mode scheduling on their sinkpads. This means that all pads linked to those pads need to be activated in pull-mode. Source pads activated in pull-mode must implement a _get_range ()-function set using gst_pad_set_getrange_function (), and that function will be called when the peer pad requests some data with gst_pad_pull_range (). The element is then responsible for seeking to the right offset and providing the requested data. Several elements can implement random access:

• Data sources, such as a file source, that can provide data from any offset with reasonable low latency.

• Filters that would like to provide a pull-mode scheduling over the whole pipeline.

• Parsers who can easily provide this by skipping a small part of their input and are thus essentially "forwarding" getrange requests literally without any own processing involved. Examples include tag readers (e.g. ID3) or single output parsers, such as a WAVE parser.

The following example will show how a _get_range ()-function can be implemented in a source element:

#include "filter.h"
static GstFlowReturn
        gst_my_filter_get_range (GstPad     * pad,
                     GstObject  * parent,
                     guint64      offset,
                     guint        length,
                     GstBuffer ** buf);

G_DEFINE_TYPE (GstMyFilter, gst_my_filter, GST_TYPE_ELEMENT);



static void
gst_my_filter_init (GstMyFilter * filter)
{

[..]

  gst_pad_set_getrange_function (filter->srcpad,
      gst_my_filter_get_range);

[..]
}

static GstFlowReturn
gst_my_filter_get_range (GstPad     * pad,
             GstObject  * parent,
             guint64      offset,
             guint        length,
             GstBuffer ** buf)
{

  GstMyFilter *filter = GST_MY_FILTER (parent);

  [.. here, you would fill *buf ..]

  return GST_FLOW_OK;
}

In practice, many elements that could theoretically do random access, may in practice often be activated in push-mode scheduling anyway, since there is no downstream element able to start its own task. Therefore, in practice, those elements should implement both a _get_range ()-function and a _chain ()-function (for filters and parsers) or a _get_range ()-function and be prepared to start their own task by providing _activate_* ()-functions (for source elements).

Caps negotiation

Caps negotiation is the act of finding a media format (GstCaps) between elements that they can handle. This process in GStreamer can in most cases find an optimal solution for the complete pipeline. In this section we explain how this works.

Caps negotiation basics

In GStreamer, negotiation of the media format always follows the following simple rules:

• A downstream element suggest提议 a format on its sinkpad and places the suggestion in the result of the CAPS query performed on the sinkpad. See also Implementing a CAPS query function.
• An upstream element decides决定 on a format. It sends the selected media format downstream on its source pad with a CAPS event. Downstream elements reconfigure重新配置 themselves to handle the media type in the CAPS event on the sinkpad.
• A downstream element can inform upstream that it would like to suggest a new format by sending a RECONFIGURE event upstream. The RECONFIGURE event simply instructs命令 an upstream element to restart the negotiation phase月相. Because the element that sent out the RECONFIGURE event is now suggesting another format, the format in the pipeline might change.

In addition to the CAPS and RECONFIGURE event and the CAPS query, there is an ACCEPT_CAPS query to quickly check if a certain caps can be accepted by an element.

All negotiation follows these simple rules. Let's take a look at some typical uses cases and how negotiation happens.

Caps negotiation use cases

In what follows we will look at some use cases for push-mode scheduling. The pull-mode scheduling negotiation phase is discussed in Pull-mode Caps negotiation and is actually similar as we will see.

Since the sink pads only suggest formats and the source pads need to decide, the most complicated复杂的 work is done in the source pads. We can identify确定 3 caps negotiation use cases for the source pads:

• Fixed negotiation. An element can output one format only. See Fixed negotiation.
• Transform可变的 negotiation. There is a (fixed) transform between the input and output format of the element, usually based on some element property. The caps that the element will produce depend on the upstream caps and the caps that the element can accept depend on the downstream caps. See Transform negotiation.
• Dynamic negotiation. An element can output many formats. See Dynamic negotiation.

Fixed negotiation

In this case, the source pad can only produce a fixed format. Usually this format is encoded inside the media. No downstream element can ask for a different format, the only way that the source pad will renegotiate is when the element decides to change the caps itself.

Elements that could implement fixed caps (on their source pads) are, in general, all elements that are not renegotiable. Examples include:

• A typefinder, since the type found is part of the actual data stream and can thus not be re-negotiated. The typefinder will look at the stream of bytes, figure out the type, send a CAPS event with the caps and then push buffers of the type.
• Pretty much all demuxers, since the contained elementary data streams are defined in the file headers, and thus not renegotiable.
• Some decoders, where the format is embedded in the data stream and not part of the peercaps and where the decoder itself is not reconfigurable, too.
• Some sources that produce a fixed format.

gst_pad_use_fixed_caps() is used on the source pad with fixed caps. As long as the pad is not negotiated, the default CAPS query will return the caps presented in the padtemplate. As soon as the pad is negotiated, the CAPS query will return the negotiated caps (and nothing else). These are the relevant code snippets for fixed caps source pads.

[..]
  pad = gst_pad_new_from_static_template (..);
  gst_pad_use_fixed_caps (pad);
[..]

The fixed caps can then be set on the pad by calling gst_pad_set_caps ().

[..]
    caps = gst_caps_new_simple ("audio/x-raw",
        "format", G_TYPE_STRING, GST_AUDIO_NE(F32),
        "rate", G_TYPE_INT, ,
        "channels", G_TYPE_INT, , NULL);
    if (!gst_pad_set_caps (pad, caps)) {
      GST_ELEMENT_ERROR (element, CORE, NEGOTIATION, (NULL),
          ("Some debug information here"));
      return GST_FLOW_ERROR;
    }
[..]

These types of elements also don't have a relation between the input format and the output format, the input caps simply don't contain the information needed to produce the output caps.

All other elements that need to be configured for the format should implement full caps negotiation, which will be explained in the next few sections.

Transform negotiation

In this negotiation technique, there is a fixed transform between the element input caps and the output caps. This transformation could be parameterized参数化的 by element properties but not by the content of the stream (see Fixed negotiation for that use-case).

The caps that the element can accept depend on the (fixed transformation) downstream caps. The caps that the element can produce depend on the (fixed transformation of) the upstream caps.

This type of element can usually set caps on its source pad from the _event() function on the sink pad when it received the CAPS event. This means that the caps transform function transforms a fixed caps into another fixed caps. Examples of elements include:

• Videobox. It adds configurable border around a video frame depending on object properties.
• Identity elements. All elements that don't change the format of the data, only the content. Video and audio effects are an example. Other examples include elements that inspect the stream.
• Some decoders and encoders, where the output format is defined by input format, like mulawdec(u-law解码) and mulawenc. These decoders usually have no headers that define the content of the stream. They are usually more like conversion elements.

Below is an example of a negotiation steps of a typical transform element. In the sink pad CAPS event handler, we compute the caps for the source pad and set those.

  [...]

static gboolean
gst_my_filter_setcaps (GstMyFilter *filter,
               GstCaps *caps)
{
  GstStructure *structure;
  int rate, channels;
  gboolean ret;
  GstCaps *outcaps;

  structure = gst_caps_get_structure (caps, 0);
  ret = gst_structure_get_int (structure, "rate", &rate);
  ret = ret && gst_structure_get_int (structure, "channels", &channels);
  if (!ret)
    return FALSE;

  outcaps = gst_caps_new_simple ("audio/x-raw",
      "format", G_TYPE_STRING, GST_AUDIO_NE(S16),
      "rate", G_TYPE_INT, rate,
      "channels", G_TYPE_INT, channels, NULL);
  ret = gst_pad_set_caps (filter->srcpad, outcaps);
  gst_caps_unref (outcaps);

  return ret;
}

static gboolean
gst_my_filter_sink_event (GstPad    *pad,
                  GstObject *parent,
                  GstEvent  *event)
{
  gboolean ret;
  GstMyFilter *filter = GST_MY_FILTER (parent);

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_CAPS:
    {
      GstCaps *caps;

      gst_event_parse_caps (event, &caps);
      ret = gst_my_filter_setcaps (filter, caps);
      break;
    }
    default:
      ret = gst_pad_event_default (pad, parent, event);
      break;
  }
  return ret;
}

  [...]

Dynamic negotiation

A last negotiation method is the most complex and powerful dynamic negotiation.

Like with the transform negotiation in Transform negotiation, dynamic negotiation will perform a transformation on the downstream/upstream caps. Unlike the transform negotiation, this transform will convert fixed caps to unfixed caps. This means that the sink pad input caps can be converted into unfixed (multiple) formats. The source pad will have to choose a format from all the possibilities. It would usually like to choose a format that requires the least amount of effort to produce but it does not have to be. The selection of the format should also depend on the caps that can be accepted downstream (see a QUERY_CAPS function in Implementing a CAPS query function).

A typical flow goes like this:

• Caps are received on the sink pad of the element.
• If the element prefers to operate in passthrough mode, check if downstream accepts the caps with the ACCEPT_CAPS query. If it does, we can complete negotiation and we can operate in passthrough mode.
• Calculate the possible caps for the source pad.
• Query the downstream peer pad for the list of possible caps.
• Select from the downstream list the first caps that you can transform to and set this as the output caps. You might have to fixate the caps to some reasonable defaults to construct fixed caps.

Examples of this type of elements include:

• Converter elements such as videoconvert, audioconvert, audioresample, videoscale, ...
• Source elements such as audiotestsrc, videotestsrc, v4l2src, pulsesrc, ...

Let's look at the example of an element that can convert between samplerates, so where input and output samplerate don't have to be the same:

static gboolean
gst_my_filter_setcaps (GstMyFilter *filter,
               GstCaps *caps)
{
  if (gst_pad_set_caps (filter->srcpad, caps)) {
    filter->passthrough = TRUE;
  } else {
    GstCaps *othercaps, *newcaps;
    GstStructure *s = gst_caps_get_structure (caps, 0), *others;

    /* no passthrough, setup internal conversion */
    gst_structure_get_int (s, "channels", &filter->channels);
    othercaps = gst_pad_get_allowed_caps (filter->srcpad);
    others = gst_caps_get_structure (othercaps, 0);
    gst_structure_set (others,
      "channels", G_TYPE_INT, filter->channels, NULL);

    /* now, the samplerate value can optionally have multiple values, so
     * we "fixate" it, which means that one fixed value is chosen */
    newcaps = gst_caps_copy_nth (othercaps, 0);
    gst_caps_unref (othercaps);
    gst_pad_fixate_caps (filter->srcpad, newcaps);
    if (!gst_pad_set_caps (filter->srcpad, newcaps))
      return FALSE;

    /* we are now set up, configure internally */
    filter->passthrough = FALSE;
    gst_structure_get_int (s, "rate", &filter->from_samplerate);
    others = gst_caps_get_structure (newcaps, 0);
    gst_structure_get_int (others, "rate", &filter->to_samplerate);
  }

  return TRUE;
}

static gboolean
gst_my_filter_sink_event (GstPad    *pad,
                  GstObject *parent,
                  GstEvent  *event)
{
  gboolean ret;
  GstMyFilter *filter = GST_MY_FILTER (parent);

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_CAPS:
    {
      GstCaps *caps;

      gst_event_parse_caps (event, &caps);
      ret = gst_my_filter_setcaps (filter, caps);
      break;
    }
    default:
      ret = gst_pad_event_default (pad, parent, event);
      break;
  }
  return ret;
}

static GstFlowReturn
gst_my_filter_chain (GstPad    *pad,
             GstObject *parent,
             GstBuffer *buf)
{
  GstMyFilter *filter = GST_MY_FILTER (parent);
  GstBuffer *out;

  /* push on if in passthrough mode */
  if (filter->passthrough)
    return gst_pad_push (filter->srcpad, buf);

  /* convert, push */
  out = gst_my_filter_convert (filter, buf);
  gst_buffer_unref (buf);

  return gst_pad_push (filter->srcpad, out);
}

Upstream caps (re)negotiation

Upstream negotiation's primary use is to renegotiate (part of) an already-negotiated pipeline to a new format. Some practical examples include to select a different video size because the size of the video window changed, and the video output itself is not capable of rescaling, or because the audio channel configuration changed.

Upstream caps renegotiation is requested by sending a GST_EVENT_RECONFIGURE event upstream. The idea is that it will instruct the upstream element to reconfigure its caps by doing a new query for the allowed caps and then choosing a new caps. The element that sends out the RECONFIGURE event would influence the selection of the new caps by returning the new preferred caps from its GST_QUERY_CAPS query function. The RECONFIGURE event will set the GST_PAD_FLAG_NEED_RECONFIGURE on all pads that it travels over.

It is important to note here that different elements actually have different responsibilities here:

• Elements that want to propose a new format upstream need to first check if the new caps are acceptable upstream with an ACCEPT_CAPS query. Then they would send a RECONFIGURE event and be prepared to answer the CAPS query with the new preferred format. It should be noted that when there is no upstream element that can (or wants) to renegotiate, the element needs to deal with the currently configured format.
• Elements that operate in transform negotiation according to Transform negotiation pass the RECONFIGURE event upstream. Because these elements simply do a fixed transform based on the upstream caps, they need to send the event upstream so that it can select a new format.
• Elements that operate in fixed negotiation (Fixed negotiation) drop the RECONFIGURE event. These elements can't reconfigure and their output caps don't depend on the upstream caps so the event can be dropped.
• Elements that can be reconfigured on the source pad (source pads implementing dynamic negotiation in Dynamic negotiation) should check its NEED_RECONFIGURE flag with gst_pad_check_reconfigure () and it should start renegotiation when the function returns TRUE.

Implementing a CAPS query function

A _query ()-function with the GST_QUERY_CAPS query type is called when a peer element would like to know which formats this pad supports, and in what order of preference. The return value should be all formats that this elements supports, taking into account limitations of peer elements further downstream or upstream, sorted by order of preference, highest preference first.

static gboolean
gst_my_filter_query (GstPad *pad, GstObject * parent, GstQuery * query)
{
  gboolean ret;
  GstMyFilter *filter = GST_MY_FILTER (parent);

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_CAPS
    {
      GstPad *otherpad;
      GstCaps *temp, *caps, *filt, *tcaps;
      gint i;

      otherpad = (pad == filter->srcpad) ? filter->sinkpad :
                                           filter->srcpad;
      caps = gst_pad_get_allowed_caps (otherpad);

      gst_query_parse_caps (query, &filt);

      /* We support *any* samplerate, indifferent from the samplerate
       * supported by the linked elements on both sides. */
      for (i = 0; i < gst_caps_get_size (caps); i++) {
        GstStructure *structure = gst_caps_get_structure (caps, i);

        gst_structure_remove_field (structure, "rate");
      }

      /* make sure we only return results that intersect our
       * padtemplate */
      tcaps = gst_pad_get_pad_template_caps (pad);
      if (tcaps) {
        temp = gst_caps_intersect (caps, tcaps);
        gst_caps_unref (caps);
        gst_caps_unref (tcaps);
        caps = temp;
      }
      /* filter against the query filter when needed */
      if (filt) {
        temp = gst_caps_intersect (caps, filt);
        gst_caps_unref (caps);
        caps = temp;
      }
      gst_query_set_caps_result (query, caps);
      gst_caps_unref (caps);
      ret = TRUE;
      break;
    }
    default:
      ret = gst_pad_query_default (pad, parent, query);
      break;
  }
  return ret;
}

Pull-mode Caps negotiation

WRITEME, the mechanism of pull-mode negotiation is not yet fully understood.

Using all the knowledge you've acquired by reading this chapter, you should be able to write an element that does correct caps negotiation. If in doubt, look at other elements of the same type in our git repository to get an idea of how they do what you want to do.

Memory allocation

Memory allocation and management are very important topics in multimedia. High definition video uses many megabytes to store one single image frame. It is important to reuse memory when possible instead of constantly allocating and freeing it.

Multimedia systems usually use special-purpose chips, such as DSPs or GPUs to perform the heavy lifting (especially for video). These special-purpose chips usually have strict requirements for the memory they operate on and how it is accessed.

This chapter talks about the memory-management features available to GStreamer plugins. We will first talk about the lowlevel GstMemory object that manages access to a piece of memory and then continue with one of it's main users, the GstBuffer, which is used to exchange data between plugins and with the application. We will also discuss the GstMeta. This object can be placed on buffers to provide extra info about it and its memory. We will also discuss the GstBufferPool, which allows to more-efficiently manage buffers of the same size.

To conclude this chapter we will take a look at the GST_QUERY_ALLOCATION query, which is used to negotiate memory management options between elements.

GstMemory

GstMemory is an object that manages a region of memory. This memory object points to a region of memory of “maxsize”. The area in this memory starting at “offset” and size “size” bytes is the accessible memory region. After a GstMemory is created its maxsize can no longer be changed, however, its "offset" and "size" can.

GstAllocator

GstMemory objects are created by a GstAllocator object. Most allocators implement the default gst_allocator_alloc() method but some might implement different ones, for example, when additional parameters are needed to allocate the specific memory.

Different allocators exist for system memory, shared memory and memory backed by a DMAbuf file descriptor. To implement support for a new kind of memory type, you must implement a new allocator object.

GstMemory API example

Data access to the memory wrapped by the GstMemory object is always protected with a gst_memory_map() and gst_memory_unmap() pair. An access mode (read/write) must be given when mapping memory. The map function returns a pointer to the valid memory region that can then be accessed according to the requested access mode.

Below is an example on creating a GstMemory object and using the gst_memory_map() to access the memory region.

[...]

  GstMemory *mem;
  GstMapInfo info;
  gint i;

  /* allocate 100 bytes */
  mem = gst_allocator_alloc (NULL, 100, NULL);

  /* get access to the memory in write mode */
  gst_memory_map (mem, &info, GST_MAP_WRITE);

  /* fill with pattern */
  for (i = 0; i < info.size; i++)
    info.data[i] = i;

  /* release memory */
  gst_memory_unmap (mem, &info);

[...]

Implementing a GstAllocator

WRITEME

GstBuffer

A GstBuffer is a lightweight object that is passed from an upstream to a downstream element and contains memory and metadata. It represents the multimedia content that is pushed to or pulled by downstream elements.

A GstBuffer contains one or more GstMemory objects. These objects hold the buffer's data.

Metadata in the buffer consists of:

• DTS and PTS timestamps. These represent the decoding and presentation timestamps of the buffer content and are used by synchronizing elements to schedule buffers. These timestamps can be GST_CLOCK_TIME_NONEwhen unknown/undefined.

• The duration of the buffer contents. This duration can be GST_CLOCK_TIME_NONE when unknown/undefined.

• Media-specific offset and offset_end values. For video this is the frame number in the stream, for audio, the sample number. Other media might use different definitions.

• Arbitrary structures via GstMeta, see below.

Writability

A GstBuffer is writable when the refcount of the object is exactly 1, meaning that only one object is holding a ref to the buffer. You can only modify the buffer when it is writable. This means that you need to call gst_buffer_make_writable() before changing the timestamps, offsets, metadata or adding and removing memory blocks.

API examples

You can create a GstBuffer with gst_buffer_new () and then you can add memory objects to it. You can alternatively use the convenience function gst_buffer_new_allocate () to perform both operations at once. It's also possible to wrap existing memory with gst_buffer_new_wrapped_full () and specify the function to call when the memory should be freed.

You can access the memory of a GstBuffer by getting and mapping the GstMemory objects individually or by using gst_buffer_map (). The latter merges all the memory into one big block and then gives you a pointer to it.

Below is an example of how to create a buffer and access its memory.

[...]
  GstBuffer *buffer;
  GstMemory *mem;
  GstMapInfo info;

  /* make empty buffer */
  buffer = gst_buffer_new ();

  /* make memory holding 100 bytes */
  mem = gst_allocator_alloc (NULL, 100, NULL);

  /* add the buffer */
  gst_buffer_append_memory (buffer, mem);

[...]

  /* get WRITE access to the memory and fill with 0xff */
  gst_buffer_map (buffer, &info, GST_MAP_WRITE);
  memset (info.data, 0xff, info.size);
  gst_buffer_unmap (buffer, &info);

[...]

  /* free the buffer */
  gst_buffer_unref (buffer);

[...]

GstMeta

With the GstMeta system you can add arbitrary structures to buffers. These structures describe extra properties of the buffer such as cropping, stride, region of interest, etc.

The metadata system separates API specification (what the metadata and its API look like) and its implementation (how it works). This makes it possible to have different implementations of the same API, for example, depending on the hardware you are running on.

API example

After allocating a new GstBuffer, you can add metadata to it with the metadata-specific API. This means that you will need to link to the header file where the metadata is defined to use its API.

By convention, a metadata API with name FooBar should provide two methods, a gst_buffer_add_foo_bar_meta ()and a gst_buffer_get_foo_bar_meta (). Both functions should return a pointer to a FooBarMeta structure that contains the metadata fields. Some of the _add_*_meta () can have extra parameters that will usually be used to configure the metadata structure for you.

Let's have a look at the metadata that is used to specify a cropping region for video frames.

#include 

[...]
  GstVideoCropMeta *meta;

  /* buffer points to a video frame, add some cropping metadata */
  meta = gst_buffer_add_video_crop_meta (buffer);

  /* configure the cropping metadata */
  meta->x = 8;
  meta->y = 8;
  meta->width = 120;
  meta->height = 80;
[...]

An element can then use the metadata on the buffer when rendering the frame like this:

#include 

[...]
  GstVideoCropMeta *meta;

  /* buffer points to a video frame, get the cropping metadata */
  meta = gst_buffer_get_video_crop_meta (buffer);

  if (meta) {
    /* render frame with cropping */
    _render_frame_cropped (buffer, meta->x, meta->y, meta->width, meta->height);
  } else {
    /* render frame */
    _render_frame (buffer);
  }
[...]

Implementing new GstMeta

In the next sections we show how you can add new metadata to the system and use it on buffers.

Define the metadata API

First we need to define what our API will look like and we have to register this API to the system. This is important because the API definition will be used when elements negotiate what kind of metadata they will exchange. The API definition also contains arbitrary tags that give hints about what the metadata contains. This is important when we see how metadata is preserved as buffers pass through the pipeline.

If you are making a new implementation of an existing API, you can skip this step and move directly to the implementation.

First we start with making the my-example-meta.h header file that will contain the definition of the API and structure for our metadata.

#include 

typedef struct _MyExampleMeta MyExampleMeta;

struct _MyExampleMeta {
  GstMeta       meta;

  gint          age;
  gchar        *name;
};

GType my_example_meta_api_get_type (void);
#define MY_EXAMPLE_META_API_TYPE (my_example_meta_api_get_type())

#define gst_buffer_get_my_example_meta(b) \
  ((MyExampleMeta*)gst_buffer_get_meta((b),MY_EXAMPLE_META_API_TYPE))

The metadata API definition consists of the definition of the structure that holds a gint and a string. The first field in the structure must be a GstMeta.

We also define a my_example_meta_api_get_type () function that will register our metadata API definition and a convenience gst_buffer_get_my_example_meta () macro that simply finds and returns the metadata with our new API.

Let's have a look at how the my_example_meta_api_get_type () function is implemented in the my-example-meta.cfile:

#include "my-example-meta.h"

GType
my_example_meta_api_get_type (void)
{
  static volatile GType type;
  static const gchar *tags[] = { "foo", "bar", NULL };

  if (g_once_init_enter (&type)) {
    GType _type = gst_meta_api_type_register ("MyExampleMetaAPI", tags);
    g_once_init_leave (&type, _type);
  }
  return type;
}

As you can see, it simply uses the gst_meta_api_type_register () function to register a name and some tags for the API. The result is a new GType pointer that defines the newly registered API.

Implementing a metadata API

Next we can make an implementation for a registered metadata API GType.

The implementation details of a metadata API are kept in a GstMetaInfo structure that you make available to the users of your metadata API implementation with a my_example_meta_get_info () function and a convenience MY_EXAMPLE_META_INFO macro. You also provide a method to add your metadata implementation to a GstBuffer. Your my-example-meta.h header file will need these additions:

[...]

/* implementation */
const GstMetaInfo *my_example_meta_get_info (void);
#define MY_EXAMPLE_META_INFO (my_example_meta_get_info())

MyExampleMeta * gst_buffer_add_my_example_meta (GstBuffer      *buffer,
                                                gint            age,
                                                const gchar    *name);

Let's have a look at how these functions are implemented in the my-example-meta.c file.

[...]

static gboolean
my_example_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer)
{
  MyExampleMeta *emeta = (MyExampleMeta *) meta;

  emeta->age = 0;
  emeta->name = NULL;

  return TRUE;
}

static gboolean
my_example_meta_transform (GstBuffer * transbuf, GstMeta * meta,
    GstBuffer * buffer, GQuark type, gpointer data)
{
  MyExampleMeta *emeta = (MyExampleMeta *) meta;

  /* we always copy no matter what transform */
  gst_buffer_add_my_example_meta (transbuf, emeta->age, emeta->name);

  return TRUE;
}

static void
my_example_meta_free (GstMeta * meta, GstBuffer * buffer)
{
  MyExampleMeta *emeta = (MyExampleMeta *) meta;

  g_free (emeta->name);
  emeta->name = NULL;
}

const GstMetaInfo *
my_example_meta_get_info (void)
{
  static const GstMetaInfo *meta_info = NULL;

  if (g_once_init_enter (&meta_info)) {
    const GstMetaInfo *mi = gst_meta_register (MY_EXAMPLE_META_API_TYPE,
        "MyExampleMeta",
        sizeof (MyExampleMeta),
        my_example_meta_init,
        my_example_meta_free,
        my_example_meta_transform);
    g_once_init_leave (&meta_info, mi);
  }
  return meta_info;
}

MyExampleMeta *
gst_buffer_add_my_example_meta (GstBuffer   *buffer,
                                gint         age,
                                const gchar *name)
{
  MyExampleMeta *meta;

  g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL);

  meta = (MyExampleMeta *) gst_buffer_add_meta (buffer,
      MY_EXAMPLE_META_INFO, NULL);

  meta->age = age;
  meta->name = g_strdup (name);

  return meta;
}

gst_meta_register () registers the implementation details, like the API that you implement and the size of the metadata structure, alongside methods to initialize and free the memory area. You can also implement a transform function that will be called when a certain transformation (identified by the quark and quark specific data) is performed on a buffer.

Lastly, you implement a gst_buffer_add_*_meta() that adds the metadata implementation to a buffer and sets the values of the metadata.

GstBufferPool

The GstBufferPool object provides a convenient base class for managing lists of reusable buffers. Essential for this object is that all the buffers have the same properties such as size, padding, metadata and alignment.

A GstBufferPool can be configured to manage a minimum and maximum amount of buffers of a specific size. It can also be configured to use a specific GstAllocator for the memory of the buffers. There is also support in the bufferpool to enable bufferpool specific options, such as adding GstMeta to the pool's buffers or enabling specific padding on the buffers' memory.

A GstBufferPool can be either inactivate or active. In the inactive state, you can configure the pool. In the active state, you can't change the configuration anymore but you can acquire and release buffers from/to the pool.

In the following sections we take a look at how you can use a GstBufferPool.

API example

There can be many different GstBufferPool implementations; they are all subclasses of the GstBufferPool base class. For this example, we will assume we somehow have access to a buffer pool, either because we created it ourselves or because we were given one as a result of the ALLOCATION query, as we will see below.

The GstBufferPool is initially in the inactive state so that we can configure it. Trying to configure a GstBufferPoolthat is not in the inactive state will fail. Likewise, trying to activate a bufferpool that is not configured will also fail.

  GstStructure *config;

[...]

  /* get config structure */
  config = gst_buffer_pool_get_config (pool);

  /* set caps, size, minimum and maximum buffers in the pool */
  gst_buffer_pool_config_set_params (config, caps, size, min, max);

  /* configure allocator and parameters */
  gst_buffer_pool_config_set_allocator (config, allocator, ¶ms);

  /* store the updated configuration again */
  gst_buffer_pool_set_config (pool, config);

[...]

The configuration of a GstBufferPool is maintained in a generic GstStructure that can be obtained with gst_buffer_pool_get_config(). Convenience methods exist to get and set the configuration options in this structure. After updating the structure, it is set as the current configuration in the GstBufferPool again withgst_buffer_pool_set_config().

The following options can be configured on a GstBufferPool:

• The caps of the buffers to allocate.

• The size of the buffers. This is the suggested size of the buffers in the pool. The pool might decide to allocate larger buffers to add padding.

• The minimum and maximum amount of buffers in the pool. When minimum is set to \> 0, the bufferpool will pre-allocate this amount of buffers. When maximum is not 0, the bufferpool will allocate up to maximum amount of buffers.

• The allocator and parameters to use. Some bufferpools might ignore the allocator and use its internal one.

• Other arbitrary bufferpool options identified with a string. a bufferpool lists the supported options withgst_buffer_pool_get_options() and you can ask if an option is supported with gst_buffer_pool_has_option(). The option can be enabled by adding it to the configuration structure withgst_buffer_pool_config_add_option (). These options are used to enable things like letting the pool set metadata on the buffers or to add extra configuration options for padding, for example.

After the configuration is set on the bufferpool, the pool can be activated with gst_buffer_pool_set_active (pool, TRUE). From that point on you can use gst_buffer_pool_acquire_buffer () to retrieve a buffer from the pool, like this:

  [...]

  GstFlowReturn ret;
  GstBuffer *buffer;

  ret = gst_buffer_pool_acquire_buffer (pool, &buffer, NULL);
  if (G_UNLIKELY (ret != GST_FLOW_OK))
    goto pool_failed;

  [...]

It is important to check the return value of the acquire function because it is possible that it fails: When your element shuts down, it will deactivate the bufferpool and then all calls to acquire will return GST_FLOW_FLUSHING.

All buffers that are acquired from the pool will have their pool member set to the original pool. When the last ref is decremented on the buffer, GStreamer will automatically call gst_buffer_pool_release_buffer() to release the buffer back to the pool. You (or any other downstream element) don't need to know if a buffer came from a pool, you can just unref it.

Implementing a new GstBufferPool

WRITEME

GST_QUERY_ALLOCATION

The ALLOCATION query is used to negotiate GstMeta, GstBufferPool and GstAllocator between elements. Negotiation of the allocation strategy is always initiated and decided by a srcpad after it has negotiated a format and before it decides to push buffers. A sinkpad can suggest an allocation strategy but it is ultimately the source pad that will decide based on the suggestions of the downstream sink pad.

The source pad will do a GST_QUERY_ALLOCATION with the negotiated caps as a parameter. This is needed so that the downstream element knows what media type is being handled. A downstream sink pad can answer the allocation query with the following results:

• An array of possible GstBufferPool suggestions with suggested size, minimum and maximum amount of buffers.

• An array of GstAllocator objects along with suggested allocation parameters such as flags, prefix, alignment and padding. These allocators can also be configured in a bufferpool when this is supported by the bufferpool.

• An array of supported GstMeta implementations along with metadata specific parameters. It is important that the upstream element knows what kind of metadata is supported downstream before it places that metadata on buffers.

When the GST_QUERY_ALLOCATION returns, the source pad will select from the available bufferpools, allocators and metadata how it will allocate buffers.

ALLOCATION query example

Below is an example of the ALLOCATION query.

#include 
#include 
#include 

  GstCaps *caps;
  GstQuery *query;
  GstStructure *structure;
  GstBufferPool *pool;
  GstStructure *config;
  guint size, min, max;

[...]

  /* find a pool for the negotiated caps now */
  query = gst_query_new_allocation (caps, TRUE);

  if (!gst_pad_peer_query (scope->srcpad, query)) {
    /* query failed, not a problem, we use the query defaults */
  }

  if (gst_query_get_n_allocation_pools (query) > 0) {
    /* we got configuration from our peer, parse them */
    gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
  } else {
    pool = NULL;
    size = 0;
    min = max = 0;
  }

  if (pool == NULL) {
    /* we did not get a pool, make one ourselves then */
    pool = gst_video_buffer_pool_new ();
  }

  config = gst_buffer_pool_get_config (pool);
  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
  gst_buffer_pool_config_set_params (config, caps, size, min, max);
  gst_buffer_pool_set_config (pool, config);

  /* and activate */
  gst_buffer_pool_set_active (pool, TRUE);

[...]

This particular implementation will make a custom GstVideoBufferPool object that is specialized in allocating video buffers. You can also enable the pool to put GstVideoMeta metadata on the buffers from the pool doing:

gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META)

The ALLOCATION query in base classes

In many base classes you will see the following virtual methods for influencing the allocation strategy:

• propose_allocation () should suggest allocation parameters for the upstream element.

• decide_allocation () should decide the allocation parameters from the suggestions received from downstream.

Implementors of these methods should modify the given GstQuery object by updating the pool options and allocation options.

Media Types and Properties

There is a very large set of possible media types that may be used to pass data between elements. Indeed, each new element that is defined may use a new data format (though unless at least one other element recognises that format, it will be most likely be useless since nothing will be able to link with it).

In order for media types to be useful, and for systems like autopluggers to work, it is necessary that all elements agree on the media type definitions, and which properties are required for each media type. The GStreamer framework itself simply provides the ability to define media types and parameters, but does not fix the meaning of media types and parameters, and does not enforce standards on the creation of new media types. This is a matter for a policy to decide, not technical systems to enforce.

For now, the policy is simple:

• Do not create a new media type if you could use one which already exists.

• If creating a new media type, discuss it first with the other GStreamer developers, on at least one of: IRC, mailing lists.

• Try to ensure that the name for a new format is as unlikely to conflict with anything else created already, and is not a more generalised name than it should be. For example: "audio/compressed" would be too generalised a name to represent audio data compressed with an mp3 codec. Instead "audio/mp3" might be an appropriate name, or "audio/compressed" could exist and have a property indicating the type of compression used.

• Ensure that, when you do create a new media type, you specify it clearly, and get it added to the list of known media types so that other developers can use the media type correctly when writing their elements.

Building a Simple Format for Testing

If you need a new format that has not yet been defined in our List of Defined Types, you will want to have some general guidelines on media type naming, properties and such. A media type would ideally be equivalent to the Mime-type defined by IANA; else, it should be in the form type/x-name, where type is the sort of data this media type handles (audio, video, ...) and name should be something specific for this specific type. Audio and video media types should try to support the general audio/video properties (see the list), and can use their own properties, too. To get an idea of what properties we think are useful, see (again) the list.

Take your time to find the right set of properties for your type. There is no reason to hurry. Also, experimenting with this is generally a good idea. Experience learns that theoretically thought-out types are good, but they still need practical use to assure that they serve their needs. Make sure that your property names do not clash with similar properties used in other types. If they match, make sure they mean the same thing; properties with different types but the same names are not allowed.

Typefind Functions and Autoplugging

With only defining the types, we're not yet there. In order for a random data file to be recognized and played back as such, we need a way of recognizing their type out of the blue. For this purpose, “typefinding” was introduced. Typefinding is the process of detecting the type of a data stream. Typefinding consists of two separate parts: first, there's an unlimited number of functions that we call typefind functions, which are each able to recognize one or more types from an input stream. Then, secondly, there's a small engine which registers and calls each of those functions. This is the typefind core. On top of this typefind core, you would normally write an autoplugger, which is able to use this type detection system to dynamically build a pipeline around an input stream. Here, we will focus only on typefind functions.

A typefind function usually lives in gst-plugins-base/gst/typefind/gsttypefindfunctions.c, unless there's a good reason (like library dependencies) to put it elsewhere. The reason for this centralization is to reduce the number of plugins that need to be loaded in order to detect a stream's type. Below is an example that will recognize AVI files, which start with a “RIFF” tag, then the size of the file and then an “AVI” tag:

static void
gst_my_typefind_function (GstTypeFind *tf,
              gpointer     data)
{
  guint8 *data = gst_type_find_peek (tf, 0, 12);

  if (data &&
      GUINT32_FROM_LE (&((guint32 *) data)[0]) == GST_MAKE_FOURCC ('R','I','F','F') &&
      GUINT32_FROM_LE (&((guint32 *) data)[2]) == GST_MAKE_FOURCC ('A','V','I',' ')) {
    gst_type_find_suggest (tf, GST_TYPE_FIND_MAXIMUM,
               gst_caps_new_simple ("video/x-msvideo", NULL));
  }
}

static gboolean
plugin_init (GstPlugin *plugin)
{
  if (!gst_type_find_register (plugin, "", GST_RANK_PRIMARY,
                   gst_my_typefind_function, "avi",
                   gst_caps_new_simple ("video/x-msvideo",
                            NULL), NULL))
    return FALSE;
}

Note that gst-plugins/gst/typefind/gsttypefindfunctions.c has some simplification macros to decrease the amount of code. Make good use of those if you want to submit typefinding patches with new typefind functions.

Autoplugging has been discussed in great detail in the Application Development Manual.

List of Defined Media Types

Below is a list of all the defined types in GStreamer. They are split up in separate tables for audio, video, container, subtitle and other types, for the sake of readability. Below each table might follow a list of notes that apply to that table. In the definition of each type, we try to follow the types and rules as defined by IANA for as far as possible.

Jump directly to a specific table:

• Table of Audio Types

• Table of Video Types

• Table of Container Types

• Table of Subtitle Types

• Table of Other Types

Note that many of the properties are not required, but rather optional properties. This means that most of these properties can be extracted from the container header, but that - in case the container header does not provide these - they can also be extracted by parsing the stream header or the stream content. The policy is that your element should provide the data that it knows about by only parsing its own content, not another element's content. Example: the AVI header provides samplerate of the contained audio stream in the header. MPEG system streams don't. This means that an AVI stream demuxer would provide samplerate as a property for MPEG audio streams, whereas an MPEG demuxer would not. A decoder needing this data would require a stream parser in between two extract this from the header or calculate it from the stream.

Table of Audio Types

Media Type	Description
All audio types.
audio/*	All audio types
channels	integer
channel-mask	bitmask
format	string
layout	string
All raw audio types.
audio/x-raw	Unstructured and uncompressed raw audio data.
All encoded audio types.
audio/x-ac3	AC-3 or A52 audio streams.
audio/x-adpcm	ADPCM Audio streams.
block_align	integer
audio/x-cinepak	Audio as provided in a Cinepak (Quicktime) stream.
audio/x-dv	Audio as provided in a Digital Video stream.
audio/x-flac	Free Lossless Audio codec (FLAC).
audio/x-gsm	Data encoded by the GSM codec.
audio/x-alaw	A-Law Audio.
audio/x-mulaw	Mu-Law Audio.
audio/x-mace	MACE Audio (used in Quicktime).
audio/mpeg	Audio data compressed using the MPEG audio encoding scheme.
framed	boolean
layer	integer
bitrate	integer
audio/x-qdm2	Data encoded by the QDM version 2 codec.
audio/x-pn-realaudio	Realmedia Audio data.
audio/x-speex	Data encoded by the Speex audio codec
audio/x-vorbis	Vorbis audio data
audio/x-wma	Windows Media Audio
audio/x-paris	Ensoniq PARIS audio
audio/x-svx	Amiga IFF / SVX8 / SV16 audio
audio/x-nist	Sphere NIST audio
audio/x-voc	Sound Blaster VOC audio
audio/x-ircam	Berkeley/IRCAM/CARL audio
audio/x-w64	Sonic Foundry's 64 bit RIFF/WAV

Table of Video Types

Media Type	Description
All video types.
video/*	All video types
height	integer
framerate	fraction
max-framerate	fraction
views	integer
interlace-mode	string
chroma-site	string
colorimetry	string
pixel-aspect-ratio	fraction
format	string
All raw video types.
video/x-raw	Unstructured and uncompressed raw video data.
All encoded video types.
video/x-3ivx	3ivx video.
video/x-divx	DivX video.
video/x-dv	Digital Video.
video/x-ffv	FFMpeg video.
video/x-h263	H-263 video.
h263version	string
video/x-h264	H-264 video.
video/x-huffyuv	Huffyuv video.
video/x-indeo	Indeo video.
video/x-intel-h263	H-263 video.
video/x-jpeg	Motion-JPEG video.
video/mpeg	MPEG video.
systemstream	boolean
video/x-msmpeg	Microsoft MPEG-4 video deviations.
video/x-msvideocodec	Microsoft Video 1 (oldish codec).
video/x-pn-realvideo	Realmedia video.
video/x-rle	RLE animation format.
depth	integer
palette_data	GstBuffer
video/x-svq	Sorensen Video.
video/x-tarkin	Tarkin video.
video/x-theora	Theora video.
video/x-vp3	VP-3 video.
video/x-wmv	Windows Media Video
video/x-xvid	XviD video.
All image types.
image/gif	Graphics Interchange Format.
image/jpeg	Joint Picture Expert Group Image.
image/png	Portable Network Graphics Image.
image/tiff	Tagged Image File Format.

Table of Container Types

Media Type	Description
video/x-ms-asf	Advanced Streaming Format (ASF).
video/x-msvideo	AVI.
video/x-dv	Digital Video.
video/x-matroska	Matroska.
video/mpeg	Motion Pictures Expert Group System Stream.
application/ogg	Ogg.
video/quicktime	Quicktime.
application/vnd.rn-realmedia	RealMedia.
audio/x-wav	WAV.

Table of Subtitle Types

Media Type	Description

Table of Other Types

Media Type	Description

Events: Seeking, Navigation and More

There are many different event types but only two ways they can travel in the pipeline: downstream or upstream. It is very important to understand how both of these methods work because if one element in the pipeline is not handling them correctly the whole event system of the pipeline is broken. We will try to explain here how these methods work and how elements are supposed to implement them.

Downstream events

Downstream events are received through the sink pad's event handler, as set using gst_pad_set_event_function ()when the pad was created.

Downstream events can travel in two ways: they can be in-band (serialised with the buffer flow) or out-of-band (travelling through the pipeline instantly, possibly not in the same thread as the streaming thread that is processing the buffers, skipping ahead of buffers being processed or queued in the pipeline). The most common downstream events (SEGMENT, CAPS, TAG, EOS) are all serialised with the buffer flow.

Here is a typical event function:

static gboolean
gst_my_filter_sink_event (GstPad  *pad, GstObject * parent, GstEvent * event)
{
  GstMyFilter *filter;
  gboolean ret;

  filter = GST_MY_FILTER (parent);
  ...

  switch (GST_EVENT_TYPE (event)) {
    case GST_EVENT_SEGMENT:
      /* maybe save and/or update the current segment (e.g. for output
       * clipping) or convert the event into one in a different format
       * (e.g. BYTES to TIME) or drop it and set a flag to send a segment
       * event in a different format later */
      ret = gst_pad_push_event (filter->src_pad, event);
      break;
    case GST_EVENT_EOS:
      /* end-of-stream, we should close down all stream leftovers here */
      gst_my_filter_stop_processing (filter);
      ret = gst_pad_push_event (filter->src_pad, event);
      break;
    case GST_EVENT_FLUSH_STOP:
      gst_my_filter_clear_temporary_buffers (filter);
      ret = gst_pad_push_event (filter->src_pad, event);
      break;
    default:
      ret = gst_pad_event_default (pad, parent, event);
      break;
  }

  ...
  return ret;
}

If your element is chain-based, you will almost always have to implement a sink event function, since that is how you are notified about segments, caps and the end of the stream.

If your element is exclusively loop-based, you may or may not want a sink event function (since the element is driving the pipeline it will know the length of the stream in advance or be notified by the flow return value of gst_pad_pull_range(). In some cases even loop-based element may receive events from upstream though (for example audio decoders with an id3demux or apedemux element in front of them, or demuxers that are being fed input from sources that send additional information about the stream in custom events, as DVD sources do).

Upstream events

Upstream events are generated by an element somewhere downstream in the pipeline (example: a video sink may generate navigation events that informs upstream elements about the current position of the mouse pointer). This may also happen indirectly on request of the application, for example when the application executes a seek on a pipeline this seek request will be passed on to a sink element which will then in turn generate an upstream seek event.

The most common upstream events are seek events, Quality-of-Service (QoS) and reconfigure events.

An upstream event can be sent using the gst_pad_send_event function. This function simply call the default event handler of that pad. The default event handler of pads is gst_pad_event_default, and it basically sends the event to the peer of the internally linked pad. So upstream events always arrive on the src pad of your element and are handled by the default event handler except if you override that handler to handle it yourself. There are some specific cases where you have to do that :

• If you have multiple sink pads in your element. In that case you will have to decide which one of the sink pads you will send the event to (if not all of them).

• If you need to handle that event locally. For example a navigation event that you will want to convert before sending it upstream, or a QoS event that you want to handle.

The processing you will do in that event handler does not really matter but there are important rules you have to absolutely respect because one broken element event handler is breaking the whole pipeline event handling. Here they are :

• Always handle events you won't handle using the default gst_pad_event_default method. This method will depending on the event, forward the event or drop it.

• If you are generating some new event based on the one you received don't forget to gst_event_unref the event you received.

• Event handler function are supposed to return TRUE or FALSE indicating if the event has been handled or not. Never simply return TRUE/FALSE in that handler except if you really know that you have handled that event.

• Remember that the event handler might be called from a different thread than the streaming thread, so make sure you use appropriate locking everywhere.

All Events Together

In this chapter follows a list of all defined events that are currently being used, plus how they should be used/interpreted. You can check the what type a certain event is using the GST_EVENT_TYPE macro (or if you need a string for debugging purposes you can use GST_EVENT_TYPE_NAME).

In this chapter, we will discuss the following events:

• Stream Start

• Caps

• Segment

• Tag (metadata)

• End of Stream (EOS)

• Table Of Contents

• Gap

• Flush Start

• Flush Stop

• Quality Of Service (QOS)

• Seek Request

• Navigation

For more comprehensive information about events and how they should be used correctly in various circumstances please consult the GStreamer design documentation. This section only gives a general overview.

Stream Start

WRITEME

Caps

The CAPS event contains the format description of the following buffers. See Caps negotiation for more information about negotiation.

Segment

A segment event is sent downstream to announce the range of valid timestamps in the stream and how they should be transformed into running-time and stream-time. A segment event must always be sent before the first buffer of data and after a flush (see above).

The first segment event is created by the element driving the pipeline, like a source operating in push-mode or a demuxer/decoder operating pull-based. This segment event then travels down the pipeline and may be transformed on the way (a decoder, for example, might receive a segment event in BYTES format and might transform this into a segment event in TIMES format based on the average bitrate).

Depending on the element type, the event can simply be forwarded using gst_pad_event_default (), or it should be parsed and a modified event should be sent on. The last is true for demuxers, which generally have a byte-to-time conversion concept. Their input is usually byte-based, so the incoming event will have an offset in byte units (GST_FORMAT_BYTES), too. Elements downstream, however, expect segment events in time units, so that it can be used to synchronize against the pipeline clock. Therefore, demuxers and similar elements should not forward the event, but parse it, free it and send a segment event (in time units, GST_FORMAT_TIME) further downstream.

The segment event is created using the function gst_event_new_segment (). See the API reference and design document for details about its parameters.

Elements parsing this event can use gst_event_parse_segment() to extract the event details. Elements may find the GstSegment API useful to keep track of the current segment (if they want to use it for output clipping, for example).

Tag (metadata)

Tagging events are being sent downstream to indicate the tags as parsed from the stream data. This is currently used to preserve tags during stream transcoding from one format to the other. Tags are discussed extensively in Tagging (Metadata and Streaminfo). Most elements will simply forward the event by calling gst_pad_event_default ().

The tag event is created using the function gst_event_new_tag (), but more often elements will send a tag event downstream that will be converted into a message on the bus by sink elements. All of these functions require a filled-in taglist as argument, which they will take ownership of.

Elements parsing this event can use the function gst_event_parse_tag () to acquire the taglist that the event contains.

End of Stream (EOS)

End-of-stream events are sent if the stream that an element sends out is finished. An element receiving this event (from upstream, so it receives it on its sinkpad) will generally just process any buffered data (if there is any) and then forward the event further downstream. The gst_pad_event_default () takes care of all this, so most elements do not need to support this event. Exceptions are elements that explicitly need to close a resource down on EOS, and N-to-1 elements. Note that the stream itself is not a resource that should be closed down on EOS! Applications might seek back to a point before EOS and continue playing again.

The EOS event has no properties, which makes it one of the simplest events in GStreamer. It is created using the gst_event_new_eos() function.

It is important to note that only elements driving the pipeline should ever send an EOS event. If your element is chain-based, it is not driving the pipeline. Chain-based elements should just return GST_FLOW_EOS from their chain function at the end of the stream (or the configured segment), the upstream element that is driving the pipeline will then take care of sending the EOS event (or alternatively post a SEGMENT_DONE message on the bus depending on the mode of operation). If you are implementing your own source element, you also do not need to ever manually send an EOS event, you should also just return GST_FLOW_EOS in your create or fill function (assuming your element derives from GstBaseSrc or GstPushSrc).

Table Of Contents

WRITEME

Gap

WRITEME

Flush Start

The flush start event is sent downstream (in push mode) or upstream (in pull mode) if all buffers and caches in the pipeline should be emptied. “Queue” elements will empty their internal list of buffers when they receive this event, for example. File sink elements (e.g. “filesink”) will flush the kernel-to-disk cache (fdatasync () or fflush ()) when they receive this event. Normally, elements receiving this event will simply just forward it, since most filter or filter-like elements don't have an internal cache of data. gst_pad_event_default () does just that, so for most elements, it is enough to forward the event using the default event handler.

As a side-effect of flushing all data from the pipeline, this event unblocks the streaming thread by making all pads reject data until they receive a Flush Stop signal (elements trying to push data will get a FLUSHING flow return and stop processing data).

The flush-start event is created with the gst_event_new_flush_start (). Like the EOS event, it has no properties. This event is usually only created by elements driving the pipeline, like source elements operating in push-mode or pull-range based demuxers/decoders.

Flush Stop

The flush-stop event is sent by an element driving the pipeline after a flush-start and tells pads and elements downstream that they should accept events and buffers again (there will be at least a SEGMENT event before any buffers first though).

If your element keeps temporary caches of stream data, it should clear them when it receives a FLUSH-STOP event (and also whenever its chain function receives a buffer with the DISCONT flag set).

The flush-stop event is created with gst_event_new_flush_stop (). It has one parameter that controls if the running-time of the pipeline should be reset to 0 or not. Normally after a flushing seek, the running_time is set back to 0.

Quality Of Service (QOS)

The QOS event contains a report about the current real-time performance of the stream. See more info in Quality Of Service (QoS).

Seek Request

Seek events are meant to request a new stream position to elements. This new position can be set in several formats (time, bytes or “default units” [a term indicating frames for video, channel-independent samples for audio, etc.]). Seeking can be done with respect to the end-of-file or start-of-file, and usually happens in upstream direction (downstream seeking is done by sending a SEGMENT event with the appropriate offsets for elements that support that, like filesink).

Elements receiving seek events should, depending on the element type, either just forward it upstream (filters, decoders), change the format in which the event is given and then forward it (demuxers), or handle the event by changing the file pointer in their internal stream resource (file sources, demuxers/decoders driving the pipeline in pull-mode) or something else.

Seek events are built up using positions in specified formats (time, bytes, units). They are created using the function gst_event_new_seek (). Note that many plugins do not support seeking from the end of the stream. An element not driving the pipeline and forwarding a seek request should not assume that the seek succeeded or actually happened, it should operate based on the SEGMENT events it receives.

Elements parsing this event can do this using gst_event_parse_seek().

Navigation

Navigation events are sent upstream by video sinks to inform upstream elements of where the mouse pointer is, if and where mouse pointer clicks have happened, or if keys have been pressed or released.

All this information is contained in the event structure which can be obtained with gst_event_get_structure ().

Check out the navigationtest element in gst-plugins-good for an idea how to extract navigation information from this event.

---

Clocking

When playing complex media, each sound and video sample must be played in a specific order at a specific time. For this purpose, GStreamer provides a synchronization mechanism.

Clocks

Time in GStreamer is defined as the value returned from a particular GstClock object from the method gst_clock_get_time ().

In a typical computer, there are many sources that can be used as a time source, e.g., the system time, soundcards, CPU performance counters, ... For this reason, there are many GstClock implementations available in GStreamer. The clock time doesn't always start from 0 or from some known value. Some clocks start counting from some known start date, other clocks start counting since last reboot, etc...

As clocks return an absolute measure of time, they are not usually used directly. Instead, differences between two clock times are used to measure elapsed time according to a clock.

Clock running-time

A clock returns the absolute-time according to that clock with gst_clock_get_time (). From the absolute-time is a running-time calculated, which is simply the difference between a previous snapshot of the absolute-time called the base-time. So:

running-time = absolute-time - base-time

A GStreamer GstPipeline object maintains a GstClock object and a base-time when it goes to the PLAYING state. The pipeline gives a handle to the selected GstClock to each element in the pipeline along with selected base-time. The pipeline will select a base-time in such a way that the running-time reflects the total time spent in the PLAYING state. As a result, when the pipeline is PAUSED, the running-time stands still.

Because all objects in the pipeline have the same clock and base-time, they can thus all calculate the running-time according to the pipeline clock.

Buffer running-time

To calculate a buffer running-time, we need a buffer timestamp and the SEGMENT event that preceded the buffer. First we can convert the SEGMENT event into a GstSegment object and then we can use thegst_segment_to_running_time () function to perform the calculation of the buffer running-time.

Synchronization is now a matter of making sure that a buffer with a certain running-time is played when the clock reaches the same running-time. Usually this task is done by sink elements. Sink also have to take into account the latency configured in the pipeline and add this to the buffer running-time before synchronizing to the pipeline clock.

Obligations of each element.

Let us clarify the contract between GStreamer and each element in the pipeline.

Non-live source elements

Non-live source elements must place a timestamp in each buffer that they deliver when this is possible. They must choose the timestamps and the values of the SEGMENT event in such a way that the running-time of the buffer starts from 0.

Some sources, such as filesrc, is not able to generate timestamps on all buffers. It can and must however create a timestamp on the first buffer (with a running-time of 0).

The source then pushes out the SEGMENT event followed by the timestamped buffers.

Live source elements

Live source elements must place a timestamp in each buffer that they deliver. They must choose the timestamps and the values of the SEGMENT event in such a way that the running-time of the buffer matches exactly the running-time of the pipeline clock when the first byte in the buffer was captured.

Parser/Decoder/Encoder elements

Parser/Decoder elements must use the incoming timestamps and transfer those to the resulting output buffers. They are allowed to interpolate or reconstruct timestamps on missing input buffers when they can.

Demuxer elements

Demuxer elements can usually set the timestamps stored inside the media file onto the outgoing buffers. They need to make sure that outgoing buffers that are to be played at the same time have the same running-time. Demuxers also need to take into account the incoming timestamps on buffers and use that to calculate an offset on the outgoing buffer timestamps.

Muxer elements

Muxer elements should use the incoming buffer running-time to mux the different streams together. They should copy the incoming running-time to the outgoing buffers.

Sink elements

If the element is intended to emit samples at a specific time (real time playing), the element should require a clock, and thus implement the method set_clock.

The sink should then make sure that the sample with running-time is played exactly when the pipeline clock reaches that running-time + latency. Some elements might use the clock API such as gst_clock_id_wait() to perform this action. Other sinks might need to use other means of scheduling timely playback of the data.

Quality Of Service (QoS)

Quality of Service in GStreamer is about measuring and adjusting the real-time performance of a pipeline. The real-time performance is always measured relative to the pipeline clock and typically happens in the sinks when they synchronize buffers against the clock.

When buffers arrive late in the sink, i.e. when their running-time is smaller than that of the clock, we say that the pipeline is having a quality of service problem. These are a few possible reasons:

• High CPU load, there is not enough CPU power to handle the stream, causing buffers to arrive late in the sink.

• Network problems

• Other resource problems such as disk load, memory bottlenecks etc

The measurements result in QOS events that aim to adjust the datarate in one or more upstream elements. Two types of adjustments can be made:

• Short time "emergency" corrections based on latest observation in the sinks.

  Long term rate corrections based on trends observed in the sinks.

It is also possible for the application to artificially introduce delay between synchronized buffers, this is called throttling. It can be used to limit or reduce the framerate, for example.

Measuring QoS

Elements that synchronize buffers on the pipeline clock will usually measure the current QoS. They will also need to keep some statistics in order to generate the QOS event.

For each buffer that arrives in the sink, the element needs to calculate how late or how early it was. This is called the jitter. Negative jitter values mean that the buffer was early, positive values mean that the buffer was late. the jitter value gives an indication of how early/late a buffer was.

A synchronizing element will also need to calculate how much time elapsed between receiving two consecutive buffers. We call this the processing time because that is the amount of time it takes for the upstream element to produce/process the buffer. We can compare this processing time to the duration of the buffer to have a measurement of how fast upstream can produce data, called the proportion. If, for example, upstream can produce a buffer in 0.5 seconds of 1 second long, it is operating at twice the required speed. If, on the other hand, it takes 2 seconds to produce a buffer with 1 seconds worth of data, upstream is producing buffers too slow and we won't be able to keep synchronization. Usually, a running average is kept of the proportion.

A synchronizing element also needs to measure its own performance in order to figure out if the performance problem is upstream of itself.

These measurements are used to construct a QOS event that is sent upstream. Note that a QoS event is sent for each buffer that arrives in the sink.

Handling QoS

An element will have to install an event function on its source pads in order to receive QOS events. Usually, the element will need to store the value of the QOS event and use them in the data processing function. The element will need to use a lock to protect these QoS values as shown in the example below. Also make sure to pass the QoS event upstream.

    [...]

    case GST_EVENT_QOS:
    {
      GstQOSType type;
      gdouble proportion;
      GstClockTimeDiff diff;
      GstClockTime timestamp;

      gst_event_parse_qos (event, &type, &proportion, &diff, ×tamp);

      GST_OBJECT_LOCK (decoder);
      priv->qos_proportion = proportion;
      priv->qos_timestamp = timestamp;
      priv->qos_diff = diff;
      GST_OBJECT_UNLOCK (decoder);

      res = gst_pad_push_event (decoder->sinkpad, event);
      break;
    }

    [...]

With the QoS values, there are two types of corrections that an element can do:

Short term correction

The timestamp and the jitter value in the QOS event can be used to perform a short term correction. If the jitter is positive, the previous buffer arrived late and we can be sure that a buffer with a timestamp < timestamp + jitter is also going to be late. We can thus drop all buffers with a timestamp less than timestamp + jitter.

If the buffer duration is known, a better estimation for the next likely timestamp as: timestamp + 2 * jitter + duration.

A possible algorithm typically looks like this:

  [...]

  GST_OBJECT_LOCK (dec);
  qos_proportion = priv->qos_proportion;
  qos_timestamp = priv->qos_timestamp;
  qos_diff = priv->qos_diff;
  GST_OBJECT_UNLOCK (dec);

  /* calculate the earliest valid timestamp */
  if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (qos_timestamp))) {
    if (G_UNLIKELY (qos_diff > 0)) {
      earliest_time = qos_timestamp + 2 * qos_diff + frame_duration;
    } else {
      earliest_time = qos_timestamp + qos_diff;
    }
  } else {
    earliest_time = GST_CLOCK_TIME_NONE;
  }

  /* compare earliest_time to running-time of next buffer */
  if (earliest_time > timestamp)
    goto drop_buffer;

  [...]

Long term correction

Long term corrections are a bit more difficult to perform. They rely on the value of the proportion in the QOS event. Elements should reduce the amount of resources they consume by the proportion field in the QoS message.

Here are some possible strategies to achieve this:

• Permanently dropping frames or reducing the CPU or bandwidth requirements of the element. Some decoders might be able to skip decoding of B frames.

• Switch to lower quality processing or reduce the algorithmic complexity. Care should be taken that this doesn't introduce disturbing visual or audible glitches.

• Switch to a lower quality source to reduce network bandwidth.

• Assign more CPU cycles to critical parts of the pipeline. This could, for example, be done by increasing the thread priority.

In all cases, elements should be prepared to go back to their normal processing rate when the proportion member in the QOS event approaches the ideal proportion of 1.0 again.

Throttling

Elements synchronizing to the clock should expose a property to configure them in throttle mode. In throttle mode, the time distance between buffers is kept to a configurable throttle interval. This means that effectively the buffer rate is limited to 1 buffer per throttle interval. This can be used to limit the framerate, for example.

When an element is configured in throttling mode (this is usually only implemented on sinks) it should produce QoS events upstream with the jitter field set to the throttle interval. This should instruct upstream elements to skip or drop the remaining buffers in the configured throttle interval.

The proportion field is set to the desired slowdown needed to get the desired throttle interval. Implementations can use the QoS Throttle type, the proportion and the jitter member to tune their implementations.

The default sink base class, has the “throttle-time” property for this feature. You can test this with: gst-launch-1.0 videotestsrc ! xvimagesink throttle-time=500000000

QoS Messages

In addition to the QOS events that are sent between elements in the pipeline, there are also QOS messages posted on the pipeline bus to inform the application of QoS decisions. The QOS message contains the timestamps of when something was dropped along with the amount of dropped vs processed items. Elements must post a QOS message under these conditions:

• The element dropped a buffer because of QoS reasons.

• An element changes its processing strategy because of QoS reasons (quality). This could include a decoder that decides to drop every B frame to increase its processing speed or an effect element switching to a lower quality algorithm.

Supporting Dynamic Parameters

Warning, this part describes 0.10 and is outdated.

Sometimes object properties are not powerful enough to control the parameters that affect the behaviour of your element. When this is the case you can mark these parameters as being Controllable. Aware applications can use the controller subsystem to dynamically adjust the property values over time.

Getting Started

The controller subsystem is contained within the gstcontroller library. You need to include the header in your element's source file:

...
#include 
#include 
...

Even though the gstcontroller library may be linked into the host application, you should make sure it is initialized in your plugin_init function:

  static gboolean
  plugin_init (GstPlugin *plugin)
  {
    ...
    /* initialize library */
    gst_controller_init (NULL, NULL);
    ...
  }

It makes no sense for all GObject parameter to be real-time controlled. Therefore the next step is to mark controllable parameters. This is done by using the special flag GST_PARAM_CONTROLLABLE. when setting up GObject params in the _class_init method.

  g_object_class_install_property (gobject_class, PROP_FREQ,
      g_param_spec_double ("freq", "Frequency", "Frequency of test signal",
          0.0, 20000.0, 440.0,
          G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));

The Data Processing Loop

In the last section we learned how to mark GObject params as controllable. Application developers can then queue parameter changes for these parameters. The approach the controller subsystem takes is to make plugins responsible for pulling the changes in. This requires just one action:

    gst_object_sync_values(element,timestamp);

This call makes all parameter-changes for the given timestamp active by adjusting the GObject properties of the element. Its up to the element to determine the synchronisation rate.

The Data Processing Loop for Video Elements

For video processing elements it is the best to synchronise for every frame. That means one would add the gst_object_sync_values() call described in the previous section to the data processing function of the element.

The Data Processing Loop for Audio Elements

For audio processing elements the case is not as easy as for video processing elements. The problem here is that audio has a much higher rate. For PAL video one will e.g. process 25 full frames per second, but for standard audio it will be 44100 samples. It is rarely useful to synchronise controllable parameters that often. The easiest solution is also to have just one synchronisation call per buffer processing. This makes the control-rate depend on the buffer size.

Elements that need a specific control-rate need to break their data processing loop to synchronise every n-samples.

Interfaces

Previously, in the chapter Adding Properties, we have introduced the concept of GObject properties of controlling an element's behaviour. This is very powerful, but it has two big disadvantages: first of all, it is too generic, and second, it isn't dynamic.

The first disadvantage is related to the customizability of the end-user interface that will be built to control the element. Some properties are more important than others. Some integer properties are better shown in a spin-button widget, whereas others would be better represented by a slider widget. Such things are not possible because the UI has no actual meaning in the application. A UI widget that represents a bitrate property is the same as a UI widget that represents the size of a video, as long as both are of the same GParamSpec type. Another problem, is that things like parameter grouping, function grouping, or parameter coupling are not really possible.

The second problem with parameters are that they are not dynamic. In many cases, the allowed values for a property are not fixed, but depend on things that can only be detected at runtime. The names of inputs for a TV card in a video4linux source element, for example, can only be retrieved from the kernel driver when we've opened the device; this only happens when the element goes into the READY state. This means that we cannot create an enum property type to show this to the user.

The solution to those problems is to create very specialized types of controls for certain often-used controls. We use the concept of interfaces to achieve this. The basis of this all is the glib GTypeInterface type. For each case where we think it's useful, we've created interfaces which can be implemented by elements at their own will.

One important note: interfaces do not replace properties. Rather, interfaces should be built next to properties. There are two important reasons for this. First of all, properties can be more easily introspected. Second, properties can be specified on the commandline (gst-launch-1.0).

How to Implement Interfaces

Implementing interfaces is initiated in the _get_type () of your element. You can register one or more interfaces after having registered the type itself. Some interfaces have dependencies on other interfaces or can only be registered by certain types of elements. You will be notified of doing that wrongly when using the element: it will quit with failed assertions, which will explain what went wrong. If it does, you need to register support for thatinterface before registering support for the interface that you're wanting to support. The example below explains how to add support for a simple interface with no further dependencies.

static void gst_my_filter_some_interface_init   (GstSomeInterface *iface);

GType
gst_my_filter_get_type (void)
{
  static GType my_filter_type = 0;

  if (!my_filter_type) {
    static const GTypeInfo my_filter_info = {
      sizeof (GstMyFilterClass),
      NULL,
      NULL,
      (GClassInitFunc) gst_my_filter_class_init,
      NULL,
      NULL,
      sizeof (GstMyFilter),
      0,
      (GInstanceInitFunc) gst_my_filter_init
    };
    static const GInterfaceInfo some_interface_info = {
      (GInterfaceInitFunc) gst_my_filter_some_interface_init,
      NULL,
      NULL
    };

    my_filter_type =
    g_type_register_static (GST_TYPE_ELEMENT,
                "GstMyFilter",
                &my_filter_info, 0);
    g_type_add_interface_static (my_filter_type,
                 GST_TYPE_SOME_INTERFACE,
                                 &some_interface_info);
  }

  return my_filter_type;
}

static void
gst_my_filter_some_interface_init (GstSomeInterface *iface)
{
  /* here, you would set virtual function pointers in the interface */
}

Or more conveniently:

static void gst_my_filter_some_interface_init   (GstSomeInterface *iface);

G_DEFINE_TYPE_WITH_CODE (GstMyFilter, gst_my_filter,GST_TYPE_ELEMENT,
     G_IMPLEMENT_INTERFACE (GST_TYPE_SOME_INTERFACE,
            gst_my_filter_some_interface_init));

URI interface

WRITEME

Color Balance Interface

WRITEME

Video Overlay Interface

The GstVideoOverlay interface is used for 2 main purposes :

• To get a grab on the Window where the video sink element is going to render. This is achieved by either being informed about the Window identifier that the video sink element generated, or by forcing the video sink element to use a specific Window identifier for rendering.

• To force a redrawing of the latest video frame the video sink element displayed on the Window. Indeed if the GstPipeline is in GST\_STATE\_PAUSED state, moving the Window around will damage its content. Application developers will want to handle the Expose events themselves and force the video sink element to refresh the Window's content.

A plugin drawing video output in a video window will need to have that window at one stage or another. Passive mode simply means that no window has been given to the plugin before that stage, so the plugin created the window by itself. In that case the plugin is responsible of destroying that window when it's not needed any more and it has to tell the applications that a window has been created so that the application can use it. This is done using the have-window-handle message that can be posted from the plugin with the gst_video_overlay_got_window_handle method.

As you probably guessed already active mode just means sending a video window to the plugin so that video output goes there. This is done using the gst_video_overlay_set_window_handle method.

It is possible to switch from one mode to another at any moment, so the plugin implementing this interface has to handle all cases. There are only 2 methods that plugins writers have to implement and they most probably look like that :

static void
gst_my_filter_set_window_handle (GstVideoOverlay *overlay, guintptr handle)
{
  GstMyFilter *my_filter = GST_MY_FILTER (overlay);

  if (my_filter->window)
    gst_my_filter_destroy_window (my_filter->window);

  my_filter->window = handle;
}

static void
gst_my_filter_xoverlay_init (GstVideoOverlayClass *iface)
{
  iface->set_window_handle = gst_my_filter_set_window_handle;
}

You will also need to use the interface methods to post messages when needed such as when receiving a CAPS event where you will know the video geometry and maybe create the window.

static MyFilterWindow *
gst_my_filter_window_create (GstMyFilter *my_filter, gint width, gint height)
{
  MyFilterWindow *window = g_new (MyFilterWindow, 1);
  ...
  gst_video_overlay_got_window_handle (GST_VIDEO_OVERLAY (my_filter), window->win);
}

/* called from the event handler for CAPS events */
static gboolean
gst_my_filter_sink_set_caps (GstMyFilter *my_filter, GstCaps *caps)
{
  gint width, height;
  gboolean ret;
  ...
  ret = gst_structure_get_int (structure, "width", &width);
  ret &= gst_structure_get_int (structure, "height", &height);
  if (!ret) return FALSE;

  gst_video_overlay_prepare_window_handle (GST_VIDEO_OVERLAY (my_filter));

  if (!my_filter->window)
    my_filter->window = gst_my_filter_create_window (my_filter, width, height);

  ...
}

Navigation Interface

WRITEME

Tagging (Metadata and Streaminfo)

Overview

Tags are pieces of information stored in a stream that are not the content itself, but they rather describe the content. Most media container formats support tagging in one way or another. Ogg uses VorbisComment for this, MP3 uses ID3, AVI and WAV use RIFF's INFO list chunk, etc. GStreamer provides a general way for elements to read tags from the stream and expose this to the user. The tags (at least the metadata) will be part of the stream inside the pipeline. The consequence of this is that transcoding of files from one format to another will automatically preserve tags, as long as the input and output format elements both support tagging.

Tags are separated in two categories in GStreamer, even though applications won't notice anything of this. The first are called metadata, the second are called streaminfo. Metadata are tags that describe the non-technical parts of stream content. They can be changed without needing to re-encode the stream completely. Examples are “author”, “title” or “album”. The container format might still need to be re-written for the tags to fit in, though. Streaminfo, on the other hand, are tags that describe the stream contents technically. To change them, the stream needs to be re-encoded. Examples are “codec” or “bitrate”. Note that some container formats (like ID3) store various streaminfo tags as metadata in the file container, which means that they can be changed so that they don't match the content in the file any more. Still, they are called metadata because technically, they can be changed without re-encoding the whole stream, even though that makes them invalid. Files with such metadata tags will have the same tag twice: once as metadata, once as streaminfo.

There is no special name for tag reading elements in GStreamer. There are specialised elements (e.g. id3demux) that do nothing besides tag reading, but any GStreamer element may extract tags while processing data, and most decoders, demuxers and parsers do.

A tag writer is called TagSetter. An element supporting both can be used in a tag editor for quick tag changing (note: in-place tag editing is still poorly supported at the time of writing and usually requires tag extraction/stripping and remuxing of the stream with new tags).

Reading Tags from Streams

The basic object for tags is a GstTagList. An element that is reading tags from a stream should create an empty taglist and fill this with individual tags. Empty tag lists can be created with gst_tag_list_new (). Then, the element can fill the list using gst_tag_list_add () or gst_tag_list_add_values (). Note that elements often read metadata as strings, but the values in the taglist might not necessarily be strings - they need to be of the type the tag was registered as (the API documentation for each predefined tag should contain the type). Be sure to use functions like gst_value_transform () to make sure that your data is of the right type. After data reading, you can send the tags downstream with the TAG event. When the TAG event reaches the sink, it will post the TAG message on the pipeline's GstBus for the application to pick up.

We currently require the core to know the GType of tags before they are being used, so all tags must be registered first. You can add new tags to the list of known tags using gst_tag_register (). If you think the tag will be useful in more cases than just your own element, it might be a good idea to add it to gsttag.c instead. That's up to you to decide. If you want to do it in your own element, it's easiest to register the tag in one of your class init functions, preferably _class_init ().

static void
gst_my_filter_class_init (GstMyFilterClass *klass)
{
[..]
  gst_tag_register ("my_tag_name", GST_TAG_FLAG_META,
            G_TYPE_STRING,
            _("my own tag"),
            _("a tag that is specific to my own element"),
            NULL);
[..]
}

Writing Tags to Streams

Tag writers are the opposite of tag readers. Tag writers only take metadata tags into account, since that's the only type of tags that have to be written into a stream. Tag writers can receive tags in three ways: internal, application and pipeline. Internal tags are tags read by the element itself, which means that the tag writer is - in that case - a tag reader, too. Application tags are tags provided to the element via the TagSetter interface (which is just a layer). Pipeline tags are tags provided to the element from within the pipeline. The element receives such tags via the GST_EVENT_TAG event, which means that tags writers should implement an event handler. The tag writer is responsible for combining all these three into one list and writing them to the output stream.

The example below will receive tags from both application and pipeline, combine them and write them to the output stream. It implements the tag setter so applications can set tags, and retrieves pipeline tags from incoming events.

Warning, this example is outdated and doesn't work with the 1.0 version of GStreamer anymore.

GType
gst_my_filter_get_type (void)
{
[..]
    static const GInterfaceInfo tag_setter_info = {
      NULL,
      NULL,
      NULL
    };
[..]
    g_type_add_interface_static (my_filter_type,
                 GST_TYPE_TAG_SETTER,
                 &tag_setter_info);
[..]
}

static void
gst_my_filter_init (GstMyFilter *filter)
{
[..]
}

/*
 * Write one tag.
 */

static void
gst_my_filter_write_tag (const GstTagList *taglist,
             const gchar      *tagname,
             gpointer          data)
{
  GstMyFilter *filter = GST_MY_FILTER (data);
  GstBuffer *buffer;
  guint num_values = gst_tag_list_get_tag_size (list, tag_name), n;
  const GValue *from;
  GValue to = { 0 };

  g_value_init (&to, G_TYPE_STRING);

  for (n = 0; n < num_values; n++) {
    guint8 * data;
    gsize size;

    from = gst_tag_list_get_value_index (taglist, tagname, n);
    g_value_transform (from, &to);

    data = g_strdup_printf ("%s:%s", tagname,
        g_value_get_string (&to));
    size = strlen (data);

    buf = gst_buffer_new_wrapped (data, size);
    gst_pad_push (filter->srcpad, buf);
  }

  g_value_unset (&to);
}

static void
gst_my_filter_task_func (GstElement *element)
{
  GstMyFilter *filter = GST_MY_FILTER (element);
  GstTagSetter *tagsetter = GST_TAG_SETTER (element);
  GstData *data;
  GstEvent *event;
  gboolean eos = FALSE;
  GstTagList *taglist = gst_tag_list_new ();

  while (!eos) {
    data = gst_pad_pull (filter->sinkpad);

    /* We're not very much interested in data right now */
    if (GST_IS_BUFFER (data))
      gst_buffer_unref (GST_BUFFER (data));
    event = GST_EVENT (data);

    switch (GST_EVENT_TYPE (event)) {
      case GST_EVENT_TAG:
        gst_tag_list_insert (taglist, gst_event_tag_get_list (event),
                 GST_TAG_MERGE_PREPEND);
        gst_event_unref (event);
        break;
      case GST_EVENT_EOS:
        eos = TRUE;
        gst_event_unref (event);
        break;
      default:
        gst_pad_event_default (filter->sinkpad, event);
        break;
    }
  }

  /* merge tags with the ones retrieved from the application */
  if ((gst_tag_setter_get_tag_list (tagsetter)) {
    gst_tag_list_insert (taglist,
             gst_tag_setter_get_tag_list (tagsetter),
             gst_tag_setter_get_tag_merge_mode (tagsetter));
  }

  /* write tags */
  gst_tag_list_foreach (taglist, gst_my_filter_write_tag, filter);

  /* signal EOS */
  gst_pad_push (filter->srcpad, gst_event_new (GST_EVENT_EOS));
}

Note that normally, elements would not read the full stream before processing tags. Rather, they would read from each sinkpad until they've received data (since tags usually come in before the first data buffer) and process that.