rdp-yami.h264 was created with this command ./yamiencode -W 1024 -H 768 -s NV12 -i /tmp/rdp.yuv -o /tmp/rdp-yami.h264 you can use vlc to play either rdp-yami.h264 or rdp-nvenc.h264 vlc rdp-yami.h264 or vlc rdp-nvenc.h264 Here is function to init the NVidia encoder Jay // // NvEncoderSession_init NvEncoderSession *NvEncoderSession_init(NV_ENCODE_API_FUNCTION_LIST *encodeApi, void *hSession, unsigned int width, unsigned int height) { NvEncoderSession *session; NVENCSTATUS result = NV_ENC_SUCCESS; // allocate session object { session = (NvEncoderSession *)malloc(sizeof(NvEncoderSession)); session->encodeApi = encodeApi; session->hSession = hSession; session->initFlag = FALSE; session->width = width; session->height = height; session->encodeIndex = 0; } // // Init Param { memset(&(session->initParams), 0, sizeof(session->initParams)); session->initParams.version = NV_ENC_INITIALIZE_PARAMS_VER; session->initParams.encodeGUID = NV_ENC_CODEC_H264_GUID; session->initParams.presetGUID = NV_ENC_PRESET_LOW_LATENCY_DEFAULT_GUID; session->initParams.encodeWidth = width; session->initParams.encodeHeight = height; session->initParams.darWidth = width; session->initParams.darHeight = height; session->initParams.frameRateNum = 30; session->initParams.frameRateDen = 1; session->initParams.enableEncodeAsync = 0; // Sync Mode session->initParams.enablePTD = 1; session->initParams.reportSliceOffsets = 0; session->initParams.enableSubFrameWrite = 0; // session->initParams.enableExternalMEHints; // session->initParams.reservedBitFields; // session->initParams.privDataSize; // session->initParams.privData; session->initParams.encodeConfig = &(session->config); session->initParams.maxEncodeWidth = width; session->initParams.maxEncodeHeight = height; // session->initParams.maxMEHintCountsPerBlock[2]; } // // Encode Config { NV_ENC_PRESET_CONFIG presetConfig; memset(&presetConfig, 0, sizeof(presetConfig)); presetConfig.version = NV_ENC_PRESET_CONFIG_VER; presetConfig.presetCfg.version = NV_ENC_CONFIG_VER; result = session->encodeApi->nvEncGetEncodePresetConfig(session->hSession, session->initParams.encodeGUID, session->initParams.presetGUID, &presetConfig); if (result != NV_ENC_SUCCESS) { printf("Error - failed to get preset config : nvEncGetEncodePresetConfig : error 0x%X\n", result); return session; } memcpy(&(session->config), &(presetConfig.presetCfg), sizeof(NV_ENC_CONFIG)); // session->config.version = NV_ENC_CONFIG_VER; session->config.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID; session->config.gopLength = NVENC_INFINITE_GOPLENGTH; session->config.frameIntervalP = 1; // 1 + B_Frame_Count // session->config.monoChromeEncoding; session->config.frameFieldMode = NV_ENC_PARAMS_FRAME_FIELD_MODE_FRAME; session->config.mvPrecision = NV_ENC_MV_PRECISION_QUARTER_PEL; session->config.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP; session->config.rcParams.averageBitRate = 5000000; session->config.rcParams.maxBitRate = 0; session->config.rcParams.vbvBufferSize = 0; session->config.rcParams.vbvInitialDelay = 0; session->config.rcParams.constQP.qpInterP = 28; session->config.rcParams.constQP.qpInterB = 28; session->config.rcParams.constQP.qpIntra = 28; session->config.encodeCodecConfig.h264Config.chromaFormatIDC = 1; session->config.encodeCodecConfig.h264Config.idrPeriod = NVENC_INFINITE_GOPLENGTH; } result = session->encodeApi->nvEncInitializeEncoder(session->hSession, &(session->initParams)); if (result != NV_ENC_SUCCESS) { printf("Error - failed to initialize encoder : nvEncInitializeEncoder : error 0x%X\n", result); return session; } // init input buffers { unsigned int index; NV_ENC_CREATE_INPUT_BUFFER inputParams; session->inputBufferFormat = NV_ENC_BUFFER_FORMAT_NV12_PL; // = YUV420, NV_ENC_BUFFER_FORMAT_YUV444_PL = YUV444 session->inputBufferCount = 4; // = 4 + B_Frame_Count session->inputBuffers = (NV_ENC_INPUT_PTR*)malloc(sizeof(NV_ENC_INPUT_PTR) * session->inputBufferCount); session->inputBufferIndex = 0; session->inputCount = 0; memset(session->inputBuffers, 0, sizeof(NV_ENC_INPUT_PTR) * session->inputBufferCount); for (index = 0; index < session->inputBufferCount; index++) { memset(&inputParams, 0, sizeof(inputParams)); inputParams.version = NV_ENC_CREATE_INPUT_BUFFER_VER; inputParams.width = session->width; inputParams.height = session->height; inputParams.memoryHeap = NV_ENC_MEMORY_HEAP_SYSMEM_CACHED; inputParams.bufferFmt = session->inputBufferFormat; result = session->encodeApi->nvEncCreateInputBuffer(session->hSession, &inputParams); if (result == NV_ENC_SUCCESS) { session->inputBuffers[index] = inputParams.inputBuffer; } else { printf("Error - failed to create input buffer : nvEncCreateInputBuffer : error 0x%X\n", result); return session; } } } // init output buffers { unsigned int index; NV_ENC_CREATE_BITSTREAM_BUFFER outputParam; session->outputBufferCount = 4; // = 4 + B_Frame_Count session->outputBuffers = (NV_ENC_OUTPUT_PTR*)malloc(sizeof(NV_ENC_OUTPUT_PTR)*session->outputBufferCount); session->outputBufferSize = 2 * 1024 * 1024; // 2 MB session->outputBufferIndex = 0; session->outputCount = 0; memset(session->outputBuffers, 0, sizeof(NV_ENC_OUTPUT_PTR) * session->outputBufferCount); for (index = 0; index < session->outputBufferCount; index++) { memset(&outputParam, 0, sizeof(outputParam)); outputParam.version = NV_ENC_CREATE_BITSTREAM_BUFFER_VER; outputParam.size = session->outputBufferSize; outputParam.memoryHeap = NV_ENC_MEMORY_HEAP_SYSMEM_CACHED; result = session->encodeApi->nvEncCreateBitstreamBuffer(session->hSession, &outputParam); if (result == NV_ENC_SUCCESS) { session->outputBuffers[index] = outputParam.bitstreamBuffer; } else { printf("Error - failed to create output buffer : nvEncCreateBitstreamBuffer : error 0x%X\n", result); return session; } } } session->initFlag = TRUE; return session; } Here is nvidia breakdown per frame frame 1 1336 frame 2 1219 frame 3 491 frame 4 696 frame 5 585 frame 6 661 frame 7 659 frame 8 86 frame 9 2721 frame 10 304 frame 11 244 frame 12 181 frame 13 190 frame 14 231 frame 15 352 frame 16 2961 frame 17 1294 frame 18 3040 frame 19 170 frame 20 1006 frame 21 131 frame 22 119 frame 23 2759 frame 24 242 frame 25 131 frame 26 7159 frame 27 215 frame 28 190 frame 29 174 frame 30 181 frame 31 2157 frame 32 154 frame 33 162 frame 34 135 frame 35 161 frame 36 132 frame 37 162 frame 38 136 frame 39 162 frame 40 133 frame 41 161 frame 42 134 frame 43 162 frame 44 133 frame 45 162 frame 46 130 frame 47 159 frame 48 134 frame 49 161 frame 50 133 frame 51 162 frame 52 134 frame 53 161 frame 54 1069 frame 55 1295 frame 56 650 frame 57 145 frame 58 916 frame 59 1016 frame 60 579 frame 61 335 frame 62 139 frame 63 225 frame 64 6089 frame 65 1259