sysmocom
/
pjproject
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support Android AMediaCodec (#2552) 

* Implement android AMediaCodec AVC/VPX codec.

* - Change "anmed" to "and_media" prefix for clearity.
- Check if encoder/decoder exists before enabling codec.

* Fix bug on enum info.

* Add support to AMRNB and AMRWB audio codec.

* Modification based on comments.

* - move VPX packetizer from codec implementation.

* - Remove unwanted changes.
- Fix bug on vpx (VP9) packetize method.

* - New method to initialize vpx packetizer config.
- Don't use pt as codec id.

* add compiler option to prioritze using software or hardware codec.

* add documentation related to packetization mode for H264.

* fix some potential compile error.

* Specify video codec PT to support #2656.
This commit is contained in:
Riza Sulistyo 2021-03-05 21:59:07 +07:00 committed by GitHub
parent 433cf50b1c
commit 940e3c0443
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 4119 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
aconfigure
View File

@ -650,6 +650,7 @@ openssl_h_present
ac_ssl_backend
ac_ssl_has_aes_gcm
ac_no_ssl
ac_no_mediacodec
ac_vpx_ldflags
ac_vpx_cflags
ac_openh264_ldflags
@ -828,6 +829,7 @@ enable_ipp
with_ipp
with_ipp_samples
with_ipp_arch
enable_android_mediacodec
with_ssl
with_gnutls
enable_darwin_ssl
@ -1502,6 +1504,8 @@ Optional Features:
package and samples location using IPPROOT and
IPPSAMPLES env var or with --with-ipp and
--with-ipp-samples options
--disable-android-mediacodec
Exclude Android MediaCodec (default: autodetect)
--disable-darwin-ssl Exclude Darwin SSL (default: autodetect)
--disable-ssl Exclude SSL support the build (default: autodetect)
@ -8060,6 +8064,73 @@ fi
# Check whether --enable-android-mediacodec was given.
if test "${enable_android_mediacodec+set}" = set; then :
enableval=$enable_android_mediacodec; if test "$enable_android_mediacodec" = "no"; then
ac_no_mediacodec=1
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: Checking if Android MediaCodec support is disabled... yes" >&5
$as_echo "Checking if Android MediaCodec support is disabled... yes" >&6; }
fi
else
case $target in
*android*)
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for AMediaCodec_createDecoderByType in -lmediandk" >&5
$as_echo_n "checking for AMediaCodec_createDecoderByType in -lmediandk... " >&6; }
if ${ac_cv_lib_mediandk_AMediaCodec_createDecoderByType+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_check_lib_save_LIBS=$LIBS
LIBS="-lmediandk $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
/* Override any GCC internal prototype to avoid an error.
Use char because int might match the return type of a GCC
builtin and then its argument prototype would still apply. */
#ifdef __cplusplus
extern "C"
#endif
char AMediaCodec_createDecoderByType ();
int
main ()
{
return AMediaCodec_createDecoderByType ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
ac_cv_lib_mediandk_AMediaCodec_createDecoderByType=yes
else
ac_cv_lib_mediandk_AMediaCodec_createDecoderByType=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mediandk_AMediaCodec_createDecoderByType" >&5
$as_echo "$ac_cv_lib_mediandk_AMediaCodec_createDecoderByType" >&6; }
if test "x$ac_cv_lib_mediandk_AMediaCodec_createDecoderByType" = xyes; then :
ac_pjmedia_has_amediacodec=1 && LIBS="-lmediandk $LIBS"
fi
if test "x$ac_pjmedia_has_amediacodec" = "x1"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: Checking if Android AMediaCodec library is available... yes" >&5
$as_echo "Checking if Android AMediaCodec library is available... yes" >&6; }
$as_echo "#define PJMEDIA_HAS_ANDROID_MEDIACODEC 1" >>confdefs.h
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: Checking if Android AMediaCodec library is available... no" >&5
$as_echo "Checking if Android AMediaCodec library is available... no" >&6; }
fi
;;
esac
fi
# Check whether --with-ssl was given.
if test "${with_ssl+set}" = set; then :

23
aconfigure.ac
View File

@ -1639,6 +1639,29 @@ else
AC_MSG_RESULT([Skipping Intel IPP settings (not wanted)])
fi
dnl # Include Android MediaCodec
AC_SUBST(ac_no_mediacodec)
AC_ARG_ENABLE(android-mediacodec,
AS_HELP_STRING([--disable-android-mediacodec],
[Exclude Android MediaCodec (default: autodetect)]),
[if test "$enable_android_mediacodec" = "no"; then
[ac_no_mediacodec=1]
AC_MSG_RESULT([Checking if Android MediaCodec support is disabled... yes])
fi],
[
case $target in
*android*)
AC_CHECK_LIB(mediandk,AMediaCodec_createDecoderByType,[ac_pjmedia_has_amediacodec=1 && LIBS="-lmediandk $LIBS"])
if test "x$ac_pjmedia_has_amediacodec" = "x1"; then
AC_MSG_RESULT([Checking if Android AMediaCodec library is available... yes])
AC_DEFINE(PJMEDIA_HAS_ANDROID_MEDIACODEC, 1)
else
AC_MSG_RESULT([Checking if Android AMediaCodec library is available... no])
fi
;;
esac
])
dnl ##########################################
dnl #

2
pjmedia/build/Makefile
View File

@ -136,7 +136,7 @@ export PJSDP_LDFLAGS += $(PJMEDIA_LDLIB) \
#
export PJMEDIA_CODEC_SRCDIR = ../src/pjmedia-codec
export PJMEDIA_CODEC_OBJS += audio_codecs.o ffmpeg_vid_codecs.o openh264.o \
h263_packetizer.o h264_packetizer.o \
h263_packetizer.o h264_packetizer.o vpx_packetizer.o \
$(OS_OBJS) $(M_OBJS) $(CC_OBJS) $(HOST_OBJS) \
ipp_codecs.o silk.o opus.o $(CODEC_OBJS) \
g7221_sdp_match.o amr_sdp_match.o passthrough.o \

6
pjmedia/build/os-auto.mak.in
View File

@ -71,6 +71,7 @@ AC_NO_G7221_CODEC=@ac_no_g7221_codec@
AC_NO_OPENCORE_AMRNB=@ac_no_opencore_amrnb@
AC_NO_OPENCORE_AMRWB=@ac_no_opencore_amrwb@
AC_NO_BCG729=@ac_no_bcg729@
AC_NO_ANDROID_MEDIACODEC=@ac_no_mediacodec@
export CODEC_OBJS=
@ -143,6 +144,11 @@ ifeq ($(AC_NO_BCG729),)
export CODEC_OBJS += bcg729.o
endif
ifeq ($(AC_NO_ANDROID_MEDIACODEC),1)
export CFLAGS += -DPJMEDIA_HAS_ANDROID_MEDIACODEC=0
else
export CODEC_OBJS += and_aud_mediacodec.o and_vid_mediacodec.o
endif
#
# SRTP

16
pjmedia/build/pjmedia_codec.vcproj
View File

@ -3631,8 +3631,12 @@
<File
RelativePath="..\src\pjmedia-codec\vpx.c"
>
</File>
<Filter
</File>
<File
RelativePath="..\src\pjmedia-codec\vpx_packetizer.c"
>
</File>
<Filter
Name="g722 Files"
>
<File
@ -3744,8 +3748,12 @@
<File
RelativePath="..\include\pjmedia-codec\vpx.h"
>
</File>
</Filter>
</File>
<File
RelativePath="..\include\pjmedia-codec\vpx_packetizer.h"
>
</File>
</Filter>
</Files>
<Globals>
</Globals>

2
pjmedia/build/pjmedia_codec.vcxproj
View File

@ -531,6 +531,7 @@
<ClCompile Include="..\src\pjmedia-codec\silk.c" />
<ClCompile Include="..\src\pjmedia-codec\speex_codec.c" />
<ClCompile Include="..\src\pjmedia-codec\vpx.c" />
<ClCompile Include="..\src\pjmedia-codec\vpx_packetizer.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\include\pjmedia-codec.h" />
@ -556,6 +557,7 @@
<ClInclude Include="..\include\pjmedia-codec\speex.h" />
<ClInclude Include="..\include\pjmedia-codec\types.h" />
<ClInclude Include="..\include\pjmedia-codec\vpx.h" />
<ClInclude Include="..\include\pjmedia-codec\vpx_packetizer.h" />
<ClInclude Include="..\src\pjmedia-codec\g722\g722_dec.h" />
<ClInclude Include="..\src\pjmedia-codec\g722\g722_enc.h" />
</ItemGroup>

6
pjmedia/build/pjmedia_codec.vcxproj.filters
View File

@ -80,6 +80,9 @@
<ClCompile Include="..\src\pjmedia-codec\vpx.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\src\pjmedia-codec\vpx_packetizer.c">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\src\pjmedia-codec\g722\g722_dec.h">
@ -157,5 +160,8 @@
<ClInclude Include="..\include\pjmedia-codec\vpx.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\include\pjmedia-codec\vpx_packetizer.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
</Project>

3
pjmedia/include/pjmedia-codec.h
View File

@ -24,7 +24,8 @@
* @file pjmedia-codec.h
* @brief Include all codecs API in PJMEDIA-CODEC
*/
#include <pjmedia-codec/and_aud_mediacodec.h>
#include <pjmedia-codec/and_vid_mediacodec.h>
#include <pjmedia-codec/audio_codecs.h>
#include <pjmedia-codec/bcg729.h>
#include <pjmedia-codec/ffmpeg_vid_codecs.h>

131
pjmedia/include/pjmedia-codec/and_aud_mediacodec.h Normal file
View File

@ -0,0 +1,131 @@
/*
* Copyright (C)2020 Teluu Inc. (http://www.teluu.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#ifndef __PJMEDIA_CODEC_AND_AUD_MEDIACODEC_H__
#define __PJMEDIA_CODEC_AND_AUD_MEDIACODEC_H__
/**
* @file and_aud_mediacodec.h
* @brief Android audio MediaCodec codecs.
*/
#include <pjmedia-codec/types.h>
/**
* @defgroup PJMEDIA_CODEC_AUD_MEDIACODEC Audio MediaCodec Codec
* @ingroup PJMEDIA_CODEC_CODECS
* @{
*
* Audio MediaCodec codec wrapper for Android.
*
* This codec wrapper contains varius codecs: i.e: AMR and AMR-WB.
*
* \section pjmedia_codec_mediacodec_AMR MediaCodec AMRNB/AMR-WB
*
* MediaCodec AMR supports 16-bit PCM audio signal with sampling rate 8000Hz,
* 20ms frame length and producing various bitrates that ranges from 4.75kbps
* to 12.2kbps.
* \subsection codec_setting Codec Settings
*
* General codec settings for this codec such as VAD and PLC can be
* manipulated through the <tt>setting</tt> field in #pjmedia_codec_param.
* Please see the documentation of #pjmedia_codec_param for more info.
* Note that MediaCodec doesn't provide internal VAD/PLC feature, they will be
* provided by PJMEDIA instead.
*
* \subsubsection bitrate Bitrate
*
* By default, encoding bitrate is 7400bps. This default setting can be
* modified using #pjmedia_codec_mgr_set_default_param() by specifying
* prefered AMR bitrate in field <tt>info::avg_bps</tt> of
* #pjmedia_codec_param. Valid bitrates could be seen in
* #pjmedia_codec_amrnb_bitrates.
*
* \subsubsection payload_format Payload Format
*
* There are two AMR payload format types, bandwidth-efficient and
* octet-aligned. Default setting is using octet-aligned. This default payload
* format can be modified using #pjmedia_codec_mgr_set_default_param().
*
* In #pjmedia_codec_param, payload format can be set by specifying SDP
* format parameters "octet-align" in the SDP "a=fmtp" attribute for
* decoding direction. Valid values are "0" (for bandwidth efficient mode)
* and "1" (for octet-aligned mode).
*
* \subsubsection mode_set Mode-Set
*
* Mode-set is used for restricting AMR modes in decoding direction.
*
* By default, no mode-set restriction applied. This default setting can be
* be modified using #pjmedia_codec_mgr_set_default_param().
*
* In #pjmedia_codec_param, mode-set could be specified via format parameters
* "mode-set" in the SDP "a=fmtp" attribute for decoding direction. Valid
* value is a comma separated list of modes from the set 0 - 7, e.g:
* "4,5,6,7". When this parameter is omitted, no mode-set restrictions applied.
*
* Here is an example of modifying AMR default codec param:
\code
pjmedia_codec_param param;
pjmedia_codec_mgr_get_default_param(.., &param);
...
// set default encoding bitrate to the highest 12.2kbps
param.info.avg_bps = 12200;
// restrict decoding bitrate to 10.2kbps and 12.2kbps only
param.setting.dec_fmtp.param[0].name = pj_str("mode-set");
param.setting.dec_fmtp.param[0].val = pj_str("6,7");
// also set to use bandwidth-efficient payload format
param.setting.dec_fmtp.param[1].name = pj_str("octet-align");
param.setting.dec_fmtp.param[1].val = pj_str("0");
param.setting.dec_fmtp.cnt = 2;
...
pjmedia_codec_mgr_set_default_param(.., &param);
\endcode
*/
PJ_BEGIN_DECL
/**
* Initialize and register Android audio MediaCodec factory to pjmedia
* endpoint.
*
* @param endpt The pjmedia endpoint.
*
* @return PJ_SUCCESS on success.
*/
PJ_DECL(pj_status_t) pjmedia_codec_and_media_aud_init( pjmedia_endpt *endpt );
/**
* Unregister Android audio MediaCodec factory from pjmedia endpoint
* and deinitialize the codec library.
*
* @return PJ_SUCCESS on success.
*/
PJ_DECL(pj_status_t) pjmedia_codec_and_media_aud_deinit( void );
PJ_END_DECL
/**
* @}
*/
#endif /* __PJMEDIA_CODEC_AND_AUD_MEDIACODEC_H__ */

77
pjmedia/include/pjmedia-codec/and_vid_mediacodec.h Normal file
View File

@ -0,0 +1,77 @@
/*
* Copyright (C)2020 Teluu Inc. (http://www.teluu.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#ifndef __PJMEDIA_CODEC_AND_VID_MEDIACODEC_H__
#define __PJMEDIA_CODEC_AND_VID_MEDIACODEC_H__
#include <pjmedia-codec/types.h>
#include <pjmedia/vid_codec.h>
/**
* @file pjmedia-codec/and_vid_mediacodec.h
* @brief Android video Mediacodec codecs.
*/
PJ_BEGIN_DECL
/**
* @defgroup PJMEDIA_HAS_ANDROID_MEDIACODEC Android Mediacodec Codec
* @ingroup PJMEDIA_CODEC_VID_CODECS
* @{
*
*
* Video MediaCodec codec wrapper for Android.
*
* This codec wrapper contains varius codecs: i.e: H.264/AVC, VP8 and VP9.
* The H.264 codec wrapper only supports non-interleaved packetization
* mode. If remote uses a different mode (e.g: single-nal), this will cause
* unpacketization issue and affect decoding process.
*/
/**
* Initialize and register Android Mediacodec video codec factory.
*
* @param mgr The video codec manager instance where this codec will
* be registered to. Specify NULL to use default instance
* (in that case, an instance of video codec manager must
* have been created beforehand).
* @param pf Pool factory.
*
* @return PJ_SUCCESS on success.
*/
PJ_DECL(pj_status_t) pjmedia_codec_and_media_vid_init(
pjmedia_vid_codec_mgr *mgr,
pj_pool_factory *pf);
/**
* Unregister Android Mediacodec video codecs factory from the video codec
* manager and deinitialize the codec library.
*
* @return PJ_SUCCESS on success.
*/
PJ_DECL(pj_status_t) pjmedia_codec_and_media_vid_deinit(void);
/**
* @}
*/
PJ_END_DECL
#endif /* __PJMEDIA_CODEC_AND_VID_MEDIACODEC_H__ */

69
pjmedia/include/pjmedia-codec/config.h
View File

@ -612,6 +612,75 @@
# define PJMEDIA_HAS_VPX_CODEC_VP9 0
#endif
/**
* Enable Android MediaCodec AMRNB codec.
*
* Default: 1
*/
#ifndef PJMEDIA_HAS_AND_MEDIA_AMRNB
# define PJMEDIA_HAS_AND_MEDIA_AMRNB 1
#endif
/**
* Enable Android MediaCodec AMRWB codec.
*
* Default: 1
*/
#ifndef PJMEDIA_HAS_AND_MEDIA_AMRWB
# define PJMEDIA_HAS_AND_MEDIA_AMRWB 1
#endif
/**
* Enable Android MediaCodec AVC/H264 codec.
*
* Default: 1
*/
#ifndef PJMEDIA_HAS_AND_MEDIA_H264
# define PJMEDIA_HAS_AND_MEDIA_H264 1
#endif
/**
* Enable Android MediaCodec VP8 codec.
*
* Default: 1
*/
#ifndef PJMEDIA_HAS_AND_MEDIA_VP8
# define PJMEDIA_HAS_AND_MEDIA_VP8 1
#endif
/**
* Enable Android MediaCodec VP9 codec.
*
* Default: 1
*/
#ifndef PJMEDIA_HAS_AND_MEDIA_VP9
# define PJMEDIA_HAS_AND_MEDIA_VP9 1
#endif
/**
* Prioritize to use software video encoder on Android MediaCodec.
* Set to 0 to prioritize Hardware encoder.
* Note: based on test, software encoder configuration provided the most stable
* configuration.
*
* Default: 1
*/
#ifndef PJMEDIA_AND_MEDIA_PRIO_SW_VID_ENC
# define PJMEDIA_AND_MEDIA_PRIO_SW_VID_ENC 1
#endif
/**
* Prioritize to use software video encoder on Android MediaCodec.
* Set to 0 to prioritize Hardware encoder.
* Note: based on test, software decoder configuration provided the most stable
* configuration.
*
* Default: 1
*/
#ifndef PJMEDIA_AND_MEDIA_PRIO_SW_VID_DEC
# define PJMEDIA_AND_MEDIA_PRIO_SW_VID_DEC 1
#endif
/**
* @}
*/

5
pjmedia/include/pjmedia-codec/config_auto.h.in
View File

@ -94,6 +94,11 @@
#undef PJMEDIA_HAS_BCG729
#endif
/* Android MediCodec codecs */
#ifndef PJMEDIA_HAS_ANDROID_MEDIACODEC
#undef PJMEDIA_HAS_ANDROID_MEDIACODEC
#endif
#endif /* __PJMEDIA_CODEC_CONFIG_AUTO_H_ */

123
pjmedia/include/pjmedia-codec/vpx_packetizer.h Normal file
View File

@ -0,0 +1,123 @@
/*
* Copyright (C) 2020 Teluu Inc. (http://www.teluu.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#ifndef __PJMEDIA_VPX_PACKETIZER_H__
#define __PJMEDIA_VPX_PACKETIZER_H__
/**
* @file vpx_packetizer.h
* @brief Packetizes VPX bitstream into RTP payload and vice versa.
*/
#include <pj/types.h>
PJ_BEGIN_DECL
/**
* Opaque declaration for VPX packetizer.
*/
typedef struct pjmedia_vpx_packetizer pjmedia_vpx_packetizer;
/**
* VPX packetizer setting.
*/
typedef struct pjmedia_vpx_packetizer_cfg
{
/**
* VPX format id.
* Default: PJMEDIA_FORMAT_VP8
*/
pj_uint32_t fmt_id;
/**
* MTU size.
* Default: PJMEDIA_MAX_VID_PAYLOAD_SIZE
*/
unsigned mtu;
}
pjmedia_vpx_packetizer_cfg;
/**
* Use this function to initialize VPX packetizer config.
*
* @param cfg The VPX packetizer config to be initialized.
*/
PJ_DECL(void) pjmedia_vpx_packetizer_cfg_default(
pjmedia_vpx_packetizer_cfg *cfg);
/**
* Create VPX packetizer.
*
* @param pool The memory pool.
* @param cfg Packetizer settings, if NULL, default setting
* will be used.
* @param p_pktz Pointer to receive the packetizer.
*
* @return PJ_SUCCESS on success.
*/
PJ_DECL(pj_status_t) pjmedia_vpx_packetizer_create(
pj_pool_t *pool,
const pjmedia_vpx_packetizer_cfg *cfg,
pjmedia_vpx_packetizer **p_pktz);
/**
* Generate an RTP payload from a VPX picture bitstream. Note that this
* function will apply in-place processing, so the bitstream may be modified
* during the packetization.
*
* @param pktz The packetizer.
* @param bits_len The length of the bitstream.
* @param bits_pos The bitstream offset to be packetized.
* @param is_keyframe The frame is keyframe.
* @param payload The output payload.
* @param payload_len The output payload length, on input it represents max
* payload length.
*
* @return PJ_SUCCESS on success.
*/
PJ_DECL(pj_status_t) pjmedia_vpx_packetize(const pjmedia_vpx_packetizer *pktz,
pj_size_t bits_len,
unsigned *bits_pos,
pj_bool_t is_keyframe,
pj_uint8_t **payload,
pj_size_t *payload_len);
/**
* Append an RTP payload to an VPX picture bitstream. Note that in case of
* noticing packet lost, application should keep calling this function with
* payload pointer set to NULL, as the packetizer need to update its internal
* state.
*
* @param pktz The packetizer.
* @param payload The payload to be unpacketized.
* @param payload_len The payload length.
* @param payload_desc_len The payload description length.
*
* @return PJ_SUCCESS on success.
*/
PJ_DECL(pj_status_t) pjmedia_vpx_unpacketize(pjmedia_vpx_packetizer *pktz,
const pj_uint8_t *payload,
pj_size_t payload_len,
unsigned *payload_desc_len);
PJ_END_DECL
#endif /* __PJMEDIA_VPX_PACKETIZER_H__ */

1376
pjmedia/src/pjmedia-codec/and_aud_mediacodec.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

1933
pjmedia/src/pjmedia-codec/and_vid_mediacodec.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

8
pjmedia/src/pjmedia-codec/audio_codecs.c
View File

@ -44,6 +44,14 @@ pjmedia_codec_register_audio_codecs(pjmedia_endpt *endpt,
PJ_ASSERT_RETURN(c->ilbc.mode==20 || c->ilbc.mode==30, PJ_EINVAL);
#if PJMEDIA_HAS_ANDROID_MEDIACODEC
/* Register Android MediaCodec */
status = pjmedia_codec_and_media_aud_init(endpt);
if (status != PJ_SUCCESS) {
return status;
}
#endif
#if PJMEDIA_HAS_PASSTHROUGH_CODECS
status = pjmedia_codec_passthrough_init2(endpt, &c->passthrough.setting);
if (status != PJ_SUCCESS)

176
pjmedia/src/pjmedia-codec/vpx.c
View File

@ -30,6 +30,8 @@
# pragma comment( lib, "vpx.lib")
#endif
#include <pjmedia-codec/vpx_packetizer.h>
/* VPX */
#include <vpx/vpx_encoder.h>
#include <vpx/vpx_decoder.h>
@ -146,6 +148,7 @@ typedef struct vpx_codec_data
pj_pool_t *pool;
pjmedia_vid_codec_param *prm;
pj_bool_t whole;
pjmedia_vpx_packetizer *pktz;
/* Encoder */
vpx_codec_iface_t *(*enc_if)();
@ -405,6 +408,7 @@ static pj_status_t vpx_codec_open(pjmedia_vid_codec *codec,
pjmedia_vid_codec_vpx_fmtp vpx_fmtp;
vpx_codec_enc_cfg_t cfg;
vpx_codec_err_t res;
pjmedia_vpx_packetizer_cfg pktz_cfg;
unsigned max_res = MAX_RX_RES;
pj_status_t status;
@ -501,6 +505,15 @@ static pj_status_t vpx_codec_open(pjmedia_vid_codec *codec,
/* Need to update param back after values are negotiated */
pj_memcpy(codec_param, param, sizeof(*codec_param));
pj_bzero(&pktz_cfg, sizeof(pktz_cfg));
pktz_cfg.mtu = param->enc_mtu;
pktz_cfg.fmt_id = param->enc_fmt.id;
status = pjmedia_vpx_packetizer_create(vpx_data->pool, &pktz_cfg,
&vpx_data->pktz);
if (status != PJ_SUCCESS)
return status;
return PJ_SUCCESS;
}
@ -640,6 +653,7 @@ static pj_status_t vpx_codec_encode_more(pjmedia_vid_codec *codec,
pj_bool_t *has_more)
{
struct vpx_codec_data *vpx_data;
pj_status_t status = PJ_SUCCESS;
PJ_ASSERT_RETURN(codec && out_size && output && has_more,
PJ_EINVAL);
@ -648,156 +662,36 @@ static pj_status_t vpx_codec_encode_more(pjmedia_vid_codec *codec,
if (vpx_data->enc_processed < vpx_data->enc_frame_size) {
unsigned payload_desc_size = 1;
unsigned max_size = vpx_data->prm->enc_mtu - payload_desc_size;
unsigned remaining_size = vpx_data->enc_frame_size -
vpx_data->enc_processed;
unsigned payload_len = PJ_MIN(remaining_size, max_size);
pj_uint8_t *p = (pj_uint8_t *)output->buf;
pj_size_t payload_len = out_size;
pj_uint8_t *p = (pj_uint8_t *)output->buf;
if (payload_len + payload_desc_size > out_size)
return PJMEDIA_CODEC_EFRMTOOSHORT;
status = pjmedia_vpx_packetize(vpx_data->pktz,
vpx_data->enc_frame_size,
&vpx_data->enc_processed,
vpx_data->enc_frame_is_keyframe,
&p,
&payload_len);
if (status != PJ_SUCCESS) {
return status;
}
pj_memcpy(p + payload_desc_size,
(vpx_data->enc_frame_whole + vpx_data->enc_processed),
payload_len);
output->size = payload_len + payload_desc_size;
output->timestamp = vpx_data->ets;
output->type = PJMEDIA_FRAME_TYPE_VIDEO;
output->bit_info = 0;
if (vpx_data->enc_frame_is_keyframe) {
output->bit_info |= PJMEDIA_VID_FRM_KEYFRAME;
}
/* Set payload header */
p[0] = 0;
if (vpx_data->prm->enc_fmt.id == PJMEDIA_FORMAT_VP8) {
/* Set N: Non-reference frame */
if (!vpx_data->enc_frame_is_keyframe) p[0] |= 0x20;
/* Set S: Start of VP8 partition. */
if (vpx_data->enc_processed == 0) p[0] |= 0x10;
} else if (vpx_data->prm->enc_fmt.id == PJMEDIA_FORMAT_VP9) {
/* Set P: Inter-picture predicted frame */
if (!vpx_data->enc_frame_is_keyframe) p[0] |= 0x40;
/* Set B: Start of a frame */
if (vpx_data->enc_processed == 0) p[0] |= 0x8;
/* Set E: End of a frame */
if (vpx_data->enc_processed + payload_len ==
vpx_data->enc_frame_size)
{
p[0] |= 0x4;
}
}
pj_memcpy(p + payload_desc_size,
(vpx_data->enc_frame_whole + vpx_data->enc_processed),
payload_len);
output->size = payload_len + payload_desc_size;
vpx_data->enc_processed += payload_len;
*has_more = (vpx_data->enc_processed < vpx_data->enc_frame_size);
}
return PJ_SUCCESS;
return status;
}
static pj_status_t vpx_unpacketize(struct vpx_codec_data *vpx_data,
const pj_uint8_t *buf,
pj_size_t packet_size,
unsigned *p_desc_len)
{
unsigned desc_len = 1;
pj_uint8_t *p = (pj_uint8_t *)buf;
#define INC_DESC_LEN() {if (++desc_len >= packet_size) return PJ_ETOOSMALL;}
if (packet_size <= desc_len) return PJ_ETOOSMALL;
if (vpx_data->prm->enc_fmt.id == PJMEDIA_FORMAT_VP8) {
/* 0 1 2 3 4 5 6 7
* +-+-+-+-+-+-+-+-+
* |X|R|N|S|R| PID | (REQUIRED)
*/
/* X: Extended control bits present. */
if (p[0] & 0x80) {
INC_DESC_LEN();
/* |I|L|T|K| RSV | */
/* I: PictureID present. */
if (p[1] & 0x80) {
INC_DESC_LEN();
/* If M bit is set, the PID field MUST contain 15 bits. */
if (p[2] & 0x80) INC_DESC_LEN();
}
/* L: TL0PICIDX present. */
if (p[1] & 0x40) INC_DESC_LEN();
/* T: TID present or K: KEYIDX present. */
if ((p[1] & 0x20) || (p[1] & 0x10)) INC_DESC_LEN();
}
} else if (vpx_data->prm->enc_fmt.id == PJMEDIA_FORMAT_VP9) {
/* 0 1 2 3 4 5 6 7
* +-+-+-+-+-+-+-+-+
* |I|P|L|F|B|E|V|-| (REQUIRED)
*/
/* I: Picture ID (PID) present. */
if (p[0] & 0x80) {
INC_DESC_LEN();
/* If M bit is set, the PID field MUST contain 15 bits. */
if (p[1] & 0x80) INC_DESC_LEN();
}
/* L: Layer indices present. */
if (p[0] & 0x20) {
INC_DESC_LEN();
if (!(p[0] & 0x10)) INC_DESC_LEN();
}
/* F: Flexible mode.
* I must also be set to 1, and if P is set, there's up to 3
* reference index.
*/
if ((p[0] & 0x10) && (p[0] & 0x80) && (p[0] & 0x40)) {
unsigned char *q = p + desc_len;
INC_DESC_LEN();
if (*q & 0x1) {
q++;
INC_DESC_LEN();
if (*q & 0x1) {
q++;
INC_DESC_LEN();
}
}
}
/* V: Scalability structure (SS) data present. */
if (p[0] & 0x2) {
unsigned char *q = p + desc_len;
unsigned N_S = (*q >> 5) + 1;
INC_DESC_LEN();
/* Y: Each spatial layer's frame resolution present. */
if (*q & 0x10) desc_len += N_S * 4;
/* G: PG description present flag. */
if (*q & 0x8) {
unsigned j;
unsigned N_G = *(p + desc_len);
INC_DESC_LEN();
for (j = 0; j< N_G; j++) {
unsigned R;
q = p + desc_len;
INC_DESC_LEN();
R = (*q & 0x0F) >> 2;
desc_len += R;
if (desc_len >= packet_size)
return PJ_ETOOSMALL;
}
}
}
}
#undef INC_DESC_LEN
*p_desc_len = desc_len;
return PJ_SUCCESS;
}
static pj_status_t vpx_codec_decode_(pjmedia_vid_codec *codec,
pj_size_t count,
pjmedia_frame packets[],
@ -841,20 +735,20 @@ static pj_status_t vpx_codec_decode_(pjmedia_vid_codec *codec,
unsigned desc_len;
unsigned packet_size = packets[i].size;
pj_status_t status;
status = vpx_unpacketize(vpx_data, packets[i].buf, packet_size,
&desc_len);
status = pjmedia_vpx_unpacketize(vpx_data->pktz, packets[i].buf,
packet_size, &desc_len);
if (status != PJ_SUCCESS) {
PJ_LOG(4,(THIS_FILE, "Unpacketize error"));
return status;
}
packet_size -= desc_len;
packet_size -= desc_len;
if (whole_len + packet_size > vpx_data->dec_buf_size) {
PJ_LOG(4,(THIS_FILE, "Decoding buffer overflow [2]"));
return PJMEDIA_CODEC_EFRMTOOSHORT;
}
pj_memcpy(vpx_data->dec_buf + whole_len,
(char *)packets[i].buf + desc_len, packet_size);
whole_len += packet_size;

225
pjmedia/src/pjmedia-codec/vpx_packetizer.c Normal file
View File

@ -0,0 +1,225 @@
/*
* Copyright (C) 2020 Teluu Inc. (http://www.teluu.com)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <pjmedia-codec/vpx_packetizer.h>
#include <pjmedia/errno.h>
#include <pjmedia/types.h>
#include <pjmedia/vid_codec_util.h>
#include <pj/assert.h>
#include <pj/errno.h>
#include <pj/log.h>
#include <pj/pool.h>
#include <pj/string.h>
#if defined(PJMEDIA_HAS_VIDEO) && (PJMEDIA_HAS_VIDEO != 0)
#define THIS_FILE "vpx_packetizer.c"
/* VPX packetizer definition */
struct pjmedia_vpx_packetizer
{
/* Current settings */
pjmedia_vpx_packetizer_cfg cfg;
};
/*
* Initialize VPX packetizer.
*/
PJ_DEF(void) pjmedia_vpx_packetizer_cfg_default(pjmedia_vpx_packetizer_cfg *cfg)
{
pj_bzero(cfg, sizeof(*cfg));
cfg->fmt_id = PJMEDIA_FORMAT_VP8;
cfg->mtu =PJMEDIA_MAX_VID_PAYLOAD_SIZE;
}
/*
* Create vpx packetizer.
*/
PJ_DEF(pj_status_t) pjmedia_vpx_packetizer_create(
pj_pool_t *pool,
const pjmedia_vpx_packetizer_cfg *cfg,
pjmedia_vpx_packetizer **p)
{
pjmedia_vpx_packetizer *p_;
PJ_ASSERT_RETURN(pool && p, PJ_EINVAL);
if (cfg && cfg->fmt_id != PJMEDIA_FORMAT_VP8 &&
cfg->fmt_id != PJMEDIA_FORMAT_VP9)
{
return PJ_ENOTSUP;
}
p_ = PJ_POOL_ZALLOC_T(pool, pjmedia_vpx_packetizer);
if (cfg) {
pj_memcpy(&p_->cfg, cfg, sizeof(*cfg));
} else {
pjmedia_vpx_packetizer_cfg_default(&p_->cfg);
}
*p = p_;
return PJ_SUCCESS;
}
/*
* Generate an RTP payload from H.264 frame bitstream, in-place processing.
*/
PJ_DEF(pj_status_t) pjmedia_vpx_packetize(const pjmedia_vpx_packetizer *pktz,
pj_size_t bits_len,
unsigned *bits_pos,
pj_bool_t is_keyframe,
pj_uint8_t **payload,
pj_size_t *payload_len)
{
unsigned payload_desc_size = 1;
unsigned max_size = pktz->cfg.mtu - payload_desc_size;
unsigned remaining_size = bits_len - *bits_pos;
unsigned out_size = *payload_len;
pj_uint8_t *bits = *payload;
*payload_len = PJ_MIN(remaining_size, max_size);
if (*payload_len + payload_desc_size > out_size)
return PJMEDIA_CODEC_EFRMTOOSHORT;
/* Set payload header */
bits[0] = 0;
if (pktz->cfg.fmt_id == PJMEDIA_FORMAT_VP8) {
/* Set N: Non-reference frame */
if (!is_keyframe) bits[0] |= 0x20;
/* Set S: Start of VP8 partition. */
if (*bits_pos == 0) bits[0] |= 0x10;
} else if (pktz->cfg.fmt_id == PJMEDIA_FORMAT_VP9) {
/* Set P: Inter-picture predicted frame */
if (!is_keyframe) bits[0] |= 0x40;
/* Set B: Start of a frame */
if (*bits_pos == 0) bits[0] |= 0x8;
/* Set E: End of a frame */
if (*bits_pos + *payload_len == bits_len) {
bits[0] |= 0x4;
}
}
return PJ_SUCCESS;
}
/*
* Append RTP payload to a VPX picture bitstream
*/
PJ_DEF(pj_status_t) pjmedia_vpx_unpacketize(pjmedia_vpx_packetizer *pktz,
const pj_uint8_t *payload,
pj_size_t payload_len,
unsigned *payload_desc_len)
{
unsigned desc_len = 1;
pj_uint8_t *p = (pj_uint8_t *)payload;
#define INC_DESC_LEN() {if (++desc_len >= payload_len) return PJ_ETOOSMALL;}
if (payload_len <= desc_len) return PJ_ETOOSMALL;
if (pktz->cfg.fmt_id == PJMEDIA_FORMAT_VP8) {
/* 0 1 2 3 4 5 6 7
* +-+-+-+-+-+-+-+-+
* |X|R|N|S|R| PID | (REQUIRED)
*/
/* X: Extended control bits present. */
if (p[0] & 0x80) {
INC_DESC_LEN();
/* |I|L|T|K| RSV | */
/* I: PictureID present. */
if (p[1] & 0x80) {
INC_DESC_LEN();
/* If M bit is set, the PID field MUST contain 15 bits. */
if (p[2] & 0x80) INC_DESC_LEN();
}
/* L: TL0PICIDX present. */
if (p[1] & 0x40) INC_DESC_LEN();
/* T: TID present or K: KEYIDX present. */
if ((p[1] & 0x20) || (p[1] & 0x10)) INC_DESC_LEN();
}
} else if (pktz->cfg.fmt_id == PJMEDIA_FORMAT_VP9) {
/* 0 1 2 3 4 5 6 7
* +-+-+-+-+-+-+-+-+
* |I|P|L|F|B|E|V|-| (REQUIRED)
*/
/* I: Picture ID (PID) present. */
if (p[0] & 0x80) {
INC_DESC_LEN();
/* If M bit is set, the PID field MUST contain 15 bits. */
if (p[1] & 0x80) INC_DESC_LEN();
}
/* L: Layer indices present. */
if (p[0] & 0x20) {
INC_DESC_LEN();
if (!(p[0] & 0x10)) INC_DESC_LEN();
}
/* F: Flexible mode.
* I must also be set to 1, and if P is set, there's up to 3
* reference index.
*/
if ((p[0] & 0x10) && (p[0] & 0x80) && (p[0] & 0x40)) {
unsigned char *q = p + desc_len;
INC_DESC_LEN();
if (*q & 0x1) {
q++;
INC_DESC_LEN();
if (*q & 0x1) {
q++;
INC_DESC_LEN();
}
}
}
/* V: Scalability structure (SS) data present. */
if (p[0] & 0x2) {
unsigned char *q = p + desc_len;
unsigned N_S = (*q >> 5) + 1;
INC_DESC_LEN();
/* Y: Each spatial layer's frame resolution present. */
if (*q & 0x10) desc_len += N_S * 4;
/* G: PG description present flag. */
if (*q & 0x8) {
unsigned j;
unsigned N_G = *(p + desc_len);
INC_DESC_LEN();
for (j = 0; j< N_G; j++) {
unsigned R;
q = p + desc_len;
INC_DESC_LEN();
R = (*q & 0x0F) >> 2;
desc_len += R;
if (desc_len >= payload_len)
return PJ_ETOOSMALL;
}
}
}
}
#undef INC_DESC_LEN
*payload_desc_len = desc_len;
return PJ_SUCCESS;
}
#endif /* PJMEDIA_HAS_VIDEO */

14
pjsip/src/pjsua-lib/pjsua_vid.c
View File

@ -85,6 +85,15 @@ pj_status_t pjsua_vid_subsys_init(void)
}
#endif
#if PJMEDIA_HAS_VIDEO && PJMEDIA_HAS_ANDROID_MEDIACODEC
status = pjmedia_codec_and_media_vid_init(NULL, &pjsua_var.cp.factory);
if (status != PJ_SUCCESS) {
pjsua_perror(THIS_FILE, "Error initializing AMediaCodec library",
status);
goto on_error;
}
#endif
#if PJMEDIA_HAS_VIDEO && PJMEDIA_HAS_OPENH264_CODEC
status = pjmedia_codec_openh264_vid_init(NULL, &pjsua_var.cp.factory);
if (status != PJ_SUCCESS) {
@ -171,6 +180,11 @@ pj_status_t pjsua_vid_subsys_destroy(void)
pjmedia_codec_vid_toolbox_deinit();
#endif
#if defined(PJMEDIA_HAS_ANDROID_MEDIACODEC) && \
PJMEDIA_HAS_ANDROID_MEDIACODEC != 0
pjmedia_codec_and_media_vid_deinit();
#endif
#if defined(PJMEDIA_HAS_OPENH264_CODEC) && PJMEDIA_HAS_OPENH264_CODEC != 0
pjmedia_codec_openh264_vid_deinit();
#endif