Architecture#
The Decoder of MediaCodec now has two methods, namely OMX and Codec2, and Android is currently fully replacing OMX.
OMX#
Open Media Acceleration (OpenMAX). A free cross-platform abstraction software layer for accelerating multimedia applications on embedded and mobile devices for capturing and rendering audio, video, and images.
Layering#
OpenMAX AL (Application Layer)#
OpenMAX-AL provides a standardized interface between applications and multimedia middleware, where multimedia middleware provides the services necessary to execute the expected API functions.
The OpenMAX AL 1.1 Reference Guide.
OpenMAX IL (Integration Layer)#
The OpenMAX IL (Integration Layer) API defines a standardized media component interface that enables developers and platform providers to integrate and communicate with multimedia codecs implemented in hardware or software.
OpenMAX DL (Development Layer)#
The OpenMAX-DL defined API includes a comprehensive set of audio, video, and image functionalities that can be implemented and optimized by chip vendors on new processors, and then used by codec vendors to write various codec functionalities.
Android Implementation#
Mainly implements the OpenMax IL layer:
- Codec drivers
- Implements components with different functionalities according to the standard header files of OpenMax IL, Android encapsulates and implements the adaptation layer interface of OpenMax for multimedia engine calls.
OpenMAX IL Interface and Implementation#
// frameworks/native/headers/media_plugin/media/openmax
OMX_Types.h: Data type definitions for OpenMax IL
OMX_Core.h: Core API for OpenMax IL
OMX_Component.h: API related to OpenMax IL components
OMX_Audio.h: Constants and data structures related to audio
OMX_IVCommon.h: Common constants and data structures for images and videos
OMX_Image.h: Constants and data structures related to images
OMX_Video.h: Constants and data structures related to videos
OMX_Other.h: Other data structures (av synchronization)
OMX_Index.h: Index of data structure definitions for OpenMax IL
OMX_ContentPipe.h: Definition of content pipes
Focus on the OMX_Component.h structure of OMX_COMPONENTTYPE
typedef struct OMX_COMPONENTTYPE
{
/* Structure size */
OMX_U32 nSize;
/* Version number */
OMX_VERSIONTYPE nVersion;
/* Private data pointer of the component */
OMX_PTR pComponentPrivate;
/* Pointer set by IL Client to store its private data, returned to all callback functions */
OMX_PTR pApplicationPrivate;
/* Get component version */
OMX_ERRORTYPE (*GetComponentVersion)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_OUT OMX_STRING pComponentName,
OMX_OUT OMX_VERSIONTYPE* pComponentVersion,
OMX_OUT OMX_VERSIONTYPE* pSpecVersion,
OMX_OUT OMX_UUIDTYPE* pComponentUUID);
/* Send command */
OMX_ERRORTYPE (*SendCommand)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_COMMANDTYPE Cmd,
OMX_IN OMX_U32 nParam1,
OMX_IN OMX_PTR pCmdData);
/* Get parameter */
OMX_ERRORTYPE (*GetParameter)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_INDEXTYPE nParamIndex,
OMX_INOUT OMX_PTR pComponentParameterStructure);
/* Set parameter */
OMX_ERRORTYPE (*SetParameter)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_INDEXTYPE nIndex,
OMX_IN OMX_PTR pComponentParameterStructure);
/* Get configuration */
OMX_ERRORTYPE (*GetConfig)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_INDEXTYPE nIndex,
OMX_INOUT OMX_PTR pComponentConfigStructure);
/* Set configuration */
OMX_ERRORTYPE (*SetConfig)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_INDEXTYPE nIndex,
OMX_IN OMX_PTR pComponentConfigStructure);
/* Convert to OMX structure index */
OMX_ERRORTYPE (*GetExtensionIndex)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_STRING cParameterName,
OMX_OUT OMX_INDEXTYPE* pIndexType);
/* Get current state of the component */
OMX_ERRORTYPE (*GetState)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_OUT OMX_STATETYPE* pState);
/* For connecting to another component */
OMX_ERRORTYPE (*ComponentTunnelRequest)(
OMX_IN OMX_HANDLETYPE hComp,
OMX_IN OMX_U32 nPort,
OMX_IN OMX_HANDLETYPE hTunneledComp,
OMX_IN OMX_U32 nTunneledPort,
OMX_INOUT OMX_TUNNELSETUPTYPE* pTunnelSetup);
/* Use buffer for a certain port */
OMX_ERRORTYPE (*UseBuffer)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_INOUT OMX_BUFFERHEADERTYPE** ppBufferHdr,
OMX_IN OMX_U32 nPortIndex,
OMX_IN OMX_PTR pAppPrivate,
OMX_IN OMX_U32 nSizeBytes,
OMX_IN OMX_U8* pBuffer);
/* Allocate buffer for a certain port */
OMX_ERRORTYPE (*AllocateBuffer)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_INOUT OMX_BUFFERHEADERTYPE** ppBuffer,
OMX_IN OMX_U32 nPortIndex,
OMX_IN OMX_PTR pAppPrivate,
OMX_IN OMX_U32 nSizeBytes);
/* Free buffer for a certain port */
OMX_ERRORTYPE (*FreeBuffer)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_U32 nPortIndex,
OMX_IN OMX_BUFFERHEADERTYPE* pBuffer);
/* Let the component consume this buffer */
OMX_ERRORTYPE (*EmptyThisBuffer)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_BUFFERHEADERTYPE* pBuffer);
/* Let the component fill this buffer */
OMX_ERRORTYPE (*FillThisBuffer)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_BUFFERHEADERTYPE* pBuffer);
/* Set callback functions */
OMX_ERRORTYPE (*SetCallbacks)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_IN OMX_CALLBACKTYPE* pCallbacks,
OMX_IN OMX_PTR pAppData);
/* Deinitialize component */
OMX_ERRORTYPE (*ComponentDeInit)(
OMX_IN OMX_HANDLETYPE hComponent);
/* Use EGL Image */
OMX_ERRORTYPE (*UseEGLImage)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_INOUT OMX_BUFFERHEADERTYPE** ppBufferHdr,
OMX_IN OMX_U32 nPortIndex,
OMX_IN OMX_PTR pAppPrivate,
OMX_IN void* eglImage);
OMX_ERRORTYPE (*ComponentRoleEnum)(
OMX_IN OMX_HANDLETYPE hComponent,
OMX_OUT OMX_U8 *cRole,
OMX_IN OMX_U32 nIndex);
} OMX_COMPONENTTYPE;
OpenMAX Adaptation Layer#
// frameworks/av/media/libmedia/include/media/IOMX.h
class IOMXNode : public IInterface {
public:
DECLARE_HYBRID_META_INTERFACE(OMXNode, IOmxNode);
typedef IOMX::buffer_id buffer_id;
virtual status_t freeNode() = 0;
virtual status_t sendCommand(
OMX_COMMANDTYPE cmd, OMX_S32 param) = 0;
virtual status_t getParameter(
OMX_INDEXTYPE index, void *params, size_t size) = 0;
virtual status_t setParameter(
OMX_INDEXTYPE index, const void *params, size_t size) = 0;
virtual status_t getConfig(
OMX_INDEXTYPE index, void *params, size_t size) = 0;
virtual status_t setConfig(
OMX_INDEXTYPE index, const void *params, size_t size) = 0;
virtual status_t setPortMode(
OMX_U32 port_index, IOMX::PortMode mode) = 0;
virtual status_t prepareForAdaptivePlayback(
OMX_U32 portIndex, OMX_BOOL enable,
OMX_U32 maxFrameWidth, OMX_U32 maxFrameHeight) = 0;
virtual status_t configureVideoTunnelMode(
OMX_U32 portIndex, OMX_BOOL tunneled,
OMX_U32 audioHwSync, native_handle_t **sidebandHandle) = 0;
virtual status_t getGraphicBufferUsage(
OMX_U32 port_index, OMX_U32* usage) = 0;
virtual status_t setInputSurface(
const sp<IOMXBufferSource> &bufferSource) = 0;
virtual status_t allocateSecureBuffer(
OMX_U32 port_index, size_t size, buffer_id *buffer,
void **buffer_data, sp<NativeHandle> *native_handle) = 0;
virtual status_t useBuffer(
OMX_U32 port_index, const OMXBuffer &omxBuf, buffer_id *buffer) = 0;
virtual status_t freeBuffer(
OMX_U32 port_index, buffer_id buffer) = 0;
virtual status_t fillBuffer(
buffer_id buffer, const OMXBuffer &omxBuf, int fenceFd = -1) = 0;
virtual status_t emptyBuffer(
buffer_id buffer, const OMXBuffer &omxBuf,
OMX_U32 flags, OMX_TICKS timestamp, int fenceFd = -1) = 0;
virtual status_t getExtensionIndex(
const char *parameter_name,
OMX_INDEXTYPE *index) = 0;
virtual status_t dispatchMessage(const omx_message &msg) = 0;
};
MediaCodec#
Initialize MediaCodec
ACodec State Diagram#
The OMX architecture is based on this state diagram to complete related operations.