PJSIP使用视频：本地预览、视频解码，对端预览

PJ_DEF(void) pjsua_media_config_default(pjsua_media_config *cfg)
{
    //....
    cfg->vid_preview_enable_native = PJ_TRUE;
}

static pj_status_t create_vid_win(pjsua_vid_win_type type,
				  const pjmedia_format *fmt,
				  pjmedia_vid_dev_index rend_id,
				  pjmedia_vid_dev_index cap_id,
				  pj_bool_t show,
                                  unsigned wnd_flags,
                                  const pjmedia_vid_dev_hwnd *wnd,
				  pjsua_vid_win_id *id)
{
    pj_bool_t enable_native_preview;
    pjsua_vid_win_id wid = PJSUA_INVALID_ID;
    pjsua_vid_win *w = NULL;
    pjmedia_vid_port_param vp_param;
    pjmedia_format fmt_;
    pj_status_t status;
    unsigned i;

    enable_native_preview = pjsua_var.media_cfg.vid_preview_enable_native;
    
    //...
    
    //...
       /*
         * Determine if the device supports native preview.
         */
        status = pjmedia_vid_dev_get_info(cap_id, &vdi);
        if (status != PJ_SUCCESS)
            goto on_error;

        if (enable_native_preview &&
             (vdi.caps & PJMEDIA_VID_DEV_CAP_INPUT_PREVIEW))
        {
            /* Device supports native preview! */
            w->is_native = PJ_TRUE;
        }
    //...

}

/**
 * Media channel.
 */
typedef struct pjmedia_vid_channel
{
    pjmedia_vid_stream     *stream;         /**< Parent stream.             */
    pjmedia_dir             dir;            /**< Channel direction.         */
    pjmedia_port            port;           /**< Port interface.            */
    unsigned                pt;             /**< Payload type.              */
    pj_bool_t               paused;         /**< Paused?.                   */
    void                   *buf;            /**< Output buffer.             */
    unsigned                buf_size;       /**< Size of output buffer.     */
    pjmedia_rtp_session     rtp;            /**< RTP session.               */
} pjmedia_vid_channel;


/**
 * This structure describes media stream.
 * A media stream is bidirectional media transmission between two endpoints.
 * It consists of two channels, i.e. encoding and decoding channels.
 * A media stream corresponds to a single "m=" line in a SDP session
 * description.
 */
struct pjmedia_vid_stream
{
    pj_pool_t               *own_pool;      /**< Internal pool.             */
    pjmedia_endpt           *endpt;         /**< Media endpoint.            */
    pjmedia_vid_codec_mgr   *codec_mgr;     /**< Codec manager.             */
    pjmedia_vid_stream_info  info;          /**< Stream info.               */
    pj_grp_lock_t           *grp_lock;      /**< Stream lock.               */

    pjmedia_vid_channel     *enc;           /**< Encoding channel.          */
    pjmedia_vid_channel     *dec;           /**< Decoding channel.          */

    pjmedia_dir              dir;           /**< Stream direction.          */
    void                    *user_data;     /**< User data.                 */
    pj_str_t                 name;          /**< Stream name                */
    pj_str_t                 cname;         /**< SDES CNAME                 */

    pjmedia_transport       *transport;     /**< Stream transport.          */
 }

音频
Stream.c (pjmedia\src\pjmedia):static pj_status_t create_channel( pj_pool_t *pool,
Stream.c (pjmedia\src\pjmedia):    status = create_channel( pool, stream, PJMEDIA_DIR_DECODING,
Stream.c (pjmedia\src\pjmedia):    status = create_channel( pool, stream, PJMEDIA_DIR_ENCODING,
视频

Vid_stream.c (pjmedia\src\pjmedia):static pj_status_t create_channel( pj_pool_t *pool,
Vid_stream.c (pjmedia\src\pjmedia):    status = create_channel( pool, stream, PJMEDIA_DIR_DECODING,
Vid_stream.c (pjmedia\src\pjmedia):    status = create_channel( pool, stream, PJMEDIA_DIR_ENCODING,

/*
 * Create stream.
 */
PJ_DEF(pj_status_t) pjmedia_vid_stream_create(
                                        pjmedia_endpt *endpt,
                                        pj_pool_t *pool,
                                        pjmedia_vid_stream_info *info,
                                        pjmedia_transport *tp,
                                        void *user_data,
                                        pjmedia_vid_stream **p_stream)

/*
 * Create media channel.
 */
static pj_status_t create_channel( pj_pool_t *pool,
                                   pjmedia_vid_stream *stream,
                                   pjmedia_dir dir,
                                   unsigned pt,
                                   const pjmedia_vid_stream_info *info,
                                   pjmedia_vid_channel **p_channel)     
{                                                                      
    /* Create RTP and RTCP sessions: */
    {
        pjmedia_rtp_session_setting settings;

        settings.flags = (pj_uint8_t)((info->rtp_seq_ts_set << 2) |
                                      (info->has_rem_ssrc << 4) | 3);
        settings.default_pt = pt;
        settings.sender_ssrc = info->ssrc;
        settings.peer_ssrc = info->rem_ssrc;
        settings.seq = info->rtp_seq;
        settings.ts = info->rtp_ts;
        status = pjmedia_rtp_session_init2(&channel->rtp, settings);
    }  

}

        /* Call sink->put_frame()
         * Note that if transmitter_cnt==0, we should still call put_frame()
         * with zero frame size, as sink may need to send keep-alive packets
         * and get timestamp update.
         */
        pj_bzero(&frame, sizeof(frame));
        frame.type = PJMEDIA_FRAME_TYPE_VIDEO;
        frame.timestamp = *now;
        if (frame_rendered) {
            frame.buf = sink->put_buf;
            frame.size = sink->put_frm_size;
        }
        status = pjmedia_port_put_frame(sink->port, &frame);
        if (frame_rendered && status != PJ_SUCCESS) {
            sink->last_err_cnt++;
            if (sink->last_err != status ||
                sink->last_err_cnt % MAX_ERR_COUNT == 0)
            {
                if (sink->last_err != status)
                    sink->last_err_cnt = 1;
                sink->last_err = status;
                PJ_PERROR(5, (THIS_FILE, status,
                              "Failed (%d time(s)) to put frame to port %d"
                              " [%s]!", sink->last_err_cnt,
                              sink->idx, sink->port->info.name.ptr));
            }
        } else {
            sink->last_err = status;
            sink->last_err_cnt = 0;
        }

/**
 * Get a frame from the port (and subsequent downstream ports).
 */
PJ_DEF(pj_status_t) pjmedia_port_get_frame( pjmedia_port *port,
                                            pjmedia_frame *frame )
{
    PJ_ASSERT_RETURN(port && frame, PJ_EINVAL);

    if (port->get_frame)
        return port->get_frame(port, frame);
    else {
        frame->type = PJMEDIA_FRAME_TYPE_NONE;
        return PJ_EINVALIDOP;
    }
}

static pj_status_t get_frame(pjmedia_port *port,
                             pjmedia_frame *frame)
{
    pjmedia_vid_stream *stream = (pjmedia_vid_stream*) port->port_data.pdata;
    pjmedia_vid_channel *channel = stream->dec;

    /* Return no frame is channel is paused */
    if (channel->paused) {
        frame->type = PJMEDIA_FRAME_TYPE_NONE;
        frame->size = 0;
        return PJ_SUCCESS;
    }

    /* Report pending events. Do not publish the event while holding the
     * stream lock as that would lead to deadlock. It should be safe to
     * operate on fmt_event without the mutex because format change normally
     * would only occur once during the start of the media.
     */
    if (stream->fmt_event.type != PJMEDIA_EVENT_NONE) {
        pjmedia_event_fmt_changed_data *fmt_chg_data;

        fmt_chg_data = &stream->fmt_event.data.fmt_changed;
        
        	#if 1	//realloc size	
        	pj_int32_t new_size = fmt_chg_data->new_fmt.det.vid.size.h*fmt_chg_data->new_fmt.det.vid.size.w*1.5;
        	if (stream->dec_max_size < new_size)
                {
                    PJ_LOG(5, (THIS_FILE, "Reallocating vid_stream dec_buffer %u --> %u",
                        (unsigned)stream->dec_max_size,
                        (unsigned)new_size));
                    pj_mutex_lock(stream->jb_mutex);
        
                    stream->dec_max_size = new_size;
                    stream->dec_frame.buf = pj_pool_alloc(stream->own_pool, stream->dec_max_size);
                    pj_mutex_unlock(stream->jb_mutex);
                }
        	#endif//realloc size
        	 
        /* Update stream info and decoding channel port info */
        if (fmt_chg_data->dir == PJMEDIA_DIR_DECODING) {
            pjmedia_format_copy(&stream->info.codec_param->dec_fmt,
                                &fmt_chg_data->new_fmt);
            pjmedia_format_copy(&stream->dec->port.info.fmt,
                                &fmt_chg_data->new_fmt);

            /* Override the framerate to be 1.5x higher in the event
             * for the renderer.
             */
            fmt_chg_data->new_fmt.det.vid.fps.num *= 3;
            fmt_chg_data->new_fmt.det.vid.fps.num /= 2;
        } else {
            pjmedia_format_copy(&stream->info.codec_param->enc_fmt,
                                &fmt_chg_data->new_fmt);
            pjmedia_format_copy(&stream->enc->port.info.fmt,
                                &fmt_chg_data->new_fmt);
        }

        dump_port_info(fmt_chg_data->dir==PJMEDIA_DIR_DECODING ?
                        stream->dec : stream->enc,
                       "changed");

        pjmedia_event_publish(NULL, port, &stream->fmt_event,
                              PJMEDIA_EVENT_PUBLISH_POST_EVENT);

        stream->fmt_event.type = PJMEDIA_EVENT_NONE;
    }

    if (stream->miss_keyframe_event.type != PJMEDIA_EVENT_NONE) {
        pjmedia_event_publish(NULL, port, &stream->miss_keyframe_event,
                              PJMEDIA_EVENT_PUBLISH_POST_EVENT);
        stream->miss_keyframe_event.type = PJMEDIA_EVENT_NONE;
    }

    pj_grp_lock_acquire( stream->grp_lock );

    if (stream->dec_frame.size == 0) {
        /* Don't have frame in buffer, try to decode one */
        if (decode_frame(stream, frame) != PJ_SUCCESS) {
            frame->type = PJMEDIA_FRAME_TYPE_NONE;
            frame->size = 0;
        }
    } else {
        if (frame->size < stream->dec_frame.size) {
            PJ_LOG(4,(stream->dec->port.info.name.ptr,
                      "Error: not enough buffer for decoded frame "
                      "(supplied=%d, required=%d)",
                      (int)frame->size, (int)stream->dec_frame.size));
            frame->type = PJMEDIA_FRAME_TYPE_NONE;
            frame->size = 0;
        } else {
            frame->type = stream->dec_frame.type;
            frame->timestamp = stream->dec_frame.timestamp;
            frame->size = stream->dec_frame.size;
            pj_memcpy(frame->buf, stream->dec_frame.buf, frame->size);
        }

        stream->dec_frame.size = 0;
    }

    pj_grp_lock_release( stream->grp_lock );

    return PJ_SUCCESS;
}

    vp_param.active = PJ_FALSE;
    
    pjmedia_vid_port_create中：
    vp->role = prm->active ? ROLE_ACTIVE : ROLE_PASSIVE;

PJ_DEF(pj_status_t) pjmedia_vid_port_create( pj_pool_t *pool,
                                             const pjmedia_vid_port_param *prm,
                                             pjmedia_vid_port **p_vid_port)
{
        ///......

    } else if (vp->role==ROLE_PASSIVE) {
        vid_pasv_port *pp;

        /* Always need to create media port for passive role */
        vp->pasv_port = pp = PJ_POOL_ZALLOC_T(pool, vid_pasv_port);
        pp->vp = vp;
        if (prm->vidparam.dir & PJMEDIA_DIR_CAPTURE)
            pp->base.get_frame = &vid_pasv_port_get_frame;
        if (prm->vidparam.dir & PJMEDIA_DIR_RENDER)
            pp->base.put_frame = &vid_pasv_port_put_frame;
        pp->base.on_destroy = &vid_pasv_port_on_destroy;
        pjmedia_port_info_init2(&pp->base.info, &vp->dev_name,
                                PJMEDIA_SIG_VID_PORT,
                                prm->vidparam.dir, &prm->vidparam.fmt);

        need_frame_buf = PJ_TRUE;
    }

    
    
    //pjsua_vid.c
static pj_status_t create_vid_win(pjsua_vid_win_type type,
                                  const pjmedia_format *fmt,
                                  pjmedia_vid_dev_index rend_id,
                                  pjmedia_vid_dev_index cap_id,
                                  pj_bool_t show,
                                  unsigned wnd_flags,
                                  const pjmedia_vid_dev_hwnd *wnd,
                                  pjsua_vid_win_id *id)


    /* Create renderer video port, only if it's not a native preview */
    if (!w->is_native) {
        status = pjmedia_vid_dev_default_param(w->pool, rend_id,
                                               &vp_param.vidparam);
        if (status != PJ_SUCCESS)
            goto on_error;

        vp_param.active = PJ_FALSE;
        vp_param.vidparam.dir = PJMEDIA_DIR_RENDER;
        vp_param.vidparam.fmt = *fmt;
        vp_param.vidparam.disp_size = fmt->det.vid.size;
        vp_param.vidparam.flags |= PJMEDIA_VID_DEV_CAP_OUTPUT_HIDE;
        vp_param.vidparam.window_hide = !show;
        vp_param.vidparam.flags |= PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW_FLAGS;
        vp_param.vidparam.window_flags = wnd_flags;
        if (wnd) {
            vp_param.vidparam.flags |= PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW;
            vp_param.vidparam.window = *wnd;
        }

        status = pjmedia_vid_port_create(w->pool, &vp_param, &w->vp_rend);
        if (status != PJ_SUCCESS)
            goto on_error;

        /* Register renderer to the video conf */
        status = pjsua_vid_conf_add_port(
                                w->pool,
                                pjmedia_vid_port_get_passive_port(w->vp_rend),
                                NULL, &w->rend_slot);
        if (status != PJ_SUCCESS)
            goto on_error;

        /* For preview window, connect capturer & renderer (via conf) */
        if (w->type == PJSUA_WND_TYPE_PREVIEW && show) {
            status = pjsua_vid_conf_connect(w->cap_slot, w->rend_slot, NULL);
            if (status != PJ_SUCCESS)
                goto on_error;
        }

        PJ_LOG(4,(THIS_FILE,
                  "%s window id %d created for cap_dev=%d rend_dev=%d",
                  pjsua_vid_win_type_name(type), wid, cap_id, rend_id));
    } else {
        PJ_LOG(4,(THIS_FILE,
                  "Preview window id %d created for cap_dev %d, "
                  "using built-in preview!",
                  wid, cap_id));
    }
}    
    
static pj_status_t vid_pasv_port_put_frame(struct pjmedia_port *this_port,
                                           pjmedia_frame *frame)
{
    struct vid_pasv_port *vpp = (struct vid_pasv_port*)this_port;
    pjmedia_vid_port *vp = vpp->vp;

    if (vp->pasv_port->is_destroying)
        return PJ_EGONE;

    handle_format_change(vp);

    if (vp->stream_role==ROLE_PASSIVE) {
        /* We are passive and the stream is passive.
         * The encoding counterpart is in vid_pasv_port_get_frame().
         */
        pj_status_t status;
        pjmedia_frame frame_;

        if (frame->size != vp->src_size) {
            if (frame->size > 0) {
                PJ_LOG(4,(THIS_FILE, "Unexpected frame size %lu, expected %lu",
                                     (unsigned long)frame->size,
                                     (unsigned long)vp->src_size));
            }

            pj_memcpy(&frame_, frame, sizeof(pjmedia_frame));
            frame_.buf = NULL;
            frame_.size = 0;

            /* Send heart beat for updating timestamp or keep-alive. */
            return pjmedia_vid_dev_stream_put_frame(vp->strm, &frame_);
        }
        
        pj_bzero(&frame_, sizeof(frame_));
        status = convert_frame(vp, frame, &frame_);
        if (status != PJ_SUCCESS)
            return status;

        //这里就输出到显示设备的put_frame了
        return pjmedia_vid_dev_stream_put_frame(vp->strm, (vp->conv.conv?
                                                           &frame_: frame));
    } else {
        /* We are passive while the stream is active so we just store the
         * frame in the buffer.
         * The encoding counterpart is located in vidstream_cap_cb()
         */
        if (frame->size == vp->src_size)
            copy_frame_to_buffer(vp, frame);
    }

    return PJ_SUCCESS;
}