1/* 2 * VDA H264 HW acceleration. 3 * 4 * copyright (c) 2011 Sebastien Zwickert 5 * 6 * This file is part of FFmpeg. 7 * 8 * FFmpeg is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; either 11 * version 2.1 of the License, or (at your option) any later version. 12 * 13 * FFmpeg is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with FFmpeg; if not, write to the Free Software 20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 21 */ 22 23#include <CoreFoundation/CFDictionary.h> 24#include <CoreFoundation/CFNumber.h> 25#include <CoreFoundation/CFData.h> 26 27#include "vda.h" 28#include "libavutil/avutil.h" 29#include "h264.h" 30 31struct vda_buffer { 32 CVPixelBufferRef cv_buffer; 33}; 34#include "internal.h" 35#include "vda_internal.h" 36 37typedef struct VDAContext { 38 // The current bitstream buffer. 39 uint8_t *bitstream; 40 41 // The current size of the bitstream. 42 int bitstream_size; 43 44 // The reference size used for fast reallocation. 45 int allocated_size; 46 47 CVImageBufferRef frame; 48} VDAContext; 49 50/* Decoder callback that adds the vda frame to the queue in display order. */ 51static void vda_decoder_callback(void *vda_hw_ctx, 52 CFDictionaryRef user_info, 53 OSStatus status, 54 uint32_t infoFlags, 55 CVImageBufferRef image_buffer) 56{ 57 struct vda_context *vda_ctx = vda_hw_ctx; 58 59 if (infoFlags & kVDADecodeInfo_FrameDropped) 60 vda_ctx->cv_buffer = NULL; 61 62 if (!image_buffer) 63 return; 64 65 if (vda_ctx->cv_pix_fmt_type != CVPixelBufferGetPixelFormatType(image_buffer)) 66 return; 67 68 vda_ctx->cv_buffer = CVPixelBufferRetain(image_buffer); 69} 70 71static int vda_sync_decode(VDAContext *ctx, struct vda_context *vda_ctx) 72{ 73 OSStatus status; 74 CFDataRef coded_frame; 75 uint32_t flush_flags = 1 << 0; ///< kVDADecoderFlush_emitFrames 76 77 coded_frame = CFDataCreate(kCFAllocatorDefault, 78 ctx->bitstream, 79 ctx->bitstream_size); 80 81 status = VDADecoderDecode(vda_ctx->decoder, 0, coded_frame, NULL); 82 83 if (kVDADecoderNoErr == status) 84 status = VDADecoderFlush(vda_ctx->decoder, flush_flags); 85 86 CFRelease(coded_frame); 87 88 return status; 89} 90 91 92static int vda_old_h264_start_frame(AVCodecContext *avctx, 93 av_unused const uint8_t *buffer, 94 av_unused uint32_t size) 95{ 96 VDAContext *vda = avctx->internal->hwaccel_priv_data; 97 struct vda_context *vda_ctx = avctx->hwaccel_context; 98 99 if (!vda_ctx->decoder) 100 return -1; 101 102 vda->bitstream_size = 0; 103 104 return 0; 105} 106 107static int vda_old_h264_decode_slice(AVCodecContext *avctx, 108 const uint8_t *buffer, 109 uint32_t size) 110{ 111 VDAContext *vda = avctx->internal->hwaccel_priv_data; 112 struct vda_context *vda_ctx = avctx->hwaccel_context; 113 void *tmp; 114 115 if (!vda_ctx->decoder) 116 return -1; 117 118 tmp = av_fast_realloc(vda->bitstream, 119 &vda->allocated_size, 120 vda->bitstream_size + size + 4); 121 if (!tmp) 122 return AVERROR(ENOMEM); 123 124 vda->bitstream = tmp; 125 126 AV_WB32(vda->bitstream + vda->bitstream_size, size); 127 memcpy(vda->bitstream + vda->bitstream_size + 4, buffer, size); 128 129 vda->bitstream_size += size + 4; 130 131 return 0; 132} 133 134static void vda_h264_release_buffer(void *opaque, uint8_t *data) 135{ 136 struct vda_buffer *context = opaque; 137 CVPixelBufferRelease(context->cv_buffer); 138 av_free(context); 139} 140 141static int vda_old_h264_end_frame(AVCodecContext *avctx) 142{ 143 H264Context *h = avctx->priv_data; 144 VDAContext *vda = avctx->internal->hwaccel_priv_data; 145 struct vda_context *vda_ctx = avctx->hwaccel_context; 146 AVFrame *frame = &h->cur_pic_ptr->f; 147 struct vda_buffer *context; 148 AVBufferRef *buffer; 149 int status; 150 151 if (!vda_ctx->decoder || !vda->bitstream) 152 return -1; 153 154 status = vda_sync_decode(vda, vda_ctx); 155 frame->data[3] = (void*)vda_ctx->cv_buffer; 156 157 if (status) 158 av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status); 159 160 if (!vda_ctx->use_ref_buffer || status) 161 return status; 162 163 context = av_mallocz(sizeof(*context)); 164 buffer = av_buffer_create(NULL, 0, vda_h264_release_buffer, context, 0); 165 if (!context || !buffer) { 166 CVPixelBufferRelease(vda_ctx->cv_buffer); 167 av_free(context); 168 return -1; 169 } 170 171 context->cv_buffer = vda_ctx->cv_buffer; 172 frame->buf[3] = buffer; 173 174 return status; 175} 176 177int ff_vda_create_decoder(struct vda_context *vda_ctx, 178 uint8_t *extradata, 179 int extradata_size) 180{ 181 OSStatus status; 182 CFNumberRef height; 183 CFNumberRef width; 184 CFNumberRef format; 185 CFDataRef avc_data; 186 CFMutableDictionaryRef config_info; 187 CFMutableDictionaryRef buffer_attributes; 188 CFMutableDictionaryRef io_surface_properties; 189 CFNumberRef cv_pix_fmt; 190 191 vda_ctx->priv_bitstream = NULL; 192 vda_ctx->priv_allocated_size = 0; 193 194 /* Each VCL NAL in the bitstream sent to the decoder 195 * is preceded by a 4 bytes length header. 196 * Change the avcC atom header if needed, to signal headers of 4 bytes. */ 197 if (extradata_size >= 4 && (extradata[4] & 0x03) != 0x03) { 198 uint8_t *rw_extradata; 199 200 if (!(rw_extradata = av_malloc(extradata_size))) 201 return AVERROR(ENOMEM); 202 203 memcpy(rw_extradata, extradata, extradata_size); 204 205 rw_extradata[4] |= 0x03; 206 207 avc_data = CFDataCreate(kCFAllocatorDefault, rw_extradata, extradata_size); 208 209 av_freep(&rw_extradata); 210 } else { 211 avc_data = CFDataCreate(kCFAllocatorDefault, extradata, extradata_size); 212 } 213 214 config_info = CFDictionaryCreateMutable(kCFAllocatorDefault, 215 4, 216 &kCFTypeDictionaryKeyCallBacks, 217 &kCFTypeDictionaryValueCallBacks); 218 219 height = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->height); 220 width = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->width); 221 format = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->format); 222 223 CFDictionarySetValue(config_info, kVDADecoderConfiguration_Height, height); 224 CFDictionarySetValue(config_info, kVDADecoderConfiguration_Width, width); 225 CFDictionarySetValue(config_info, kVDADecoderConfiguration_SourceFormat, format); 226 CFDictionarySetValue(config_info, kVDADecoderConfiguration_avcCData, avc_data); 227 228 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault, 229 2, 230 &kCFTypeDictionaryKeyCallBacks, 231 &kCFTypeDictionaryValueCallBacks); 232 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault, 233 0, 234 &kCFTypeDictionaryKeyCallBacks, 235 &kCFTypeDictionaryValueCallBacks); 236 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, 237 kCFNumberSInt32Type, 238 &vda_ctx->cv_pix_fmt_type); 239 CFDictionarySetValue(buffer_attributes, 240 kCVPixelBufferPixelFormatTypeKey, 241 cv_pix_fmt); 242 CFDictionarySetValue(buffer_attributes, 243 kCVPixelBufferIOSurfacePropertiesKey, 244 io_surface_properties); 245 246 status = VDADecoderCreate(config_info, 247 buffer_attributes, 248 (VDADecoderOutputCallback *)vda_decoder_callback, 249 vda_ctx, 250 &vda_ctx->decoder); 251 252 CFRelease(height); 253 CFRelease(width); 254 CFRelease(format); 255 CFRelease(avc_data); 256 CFRelease(config_info); 257 CFRelease(io_surface_properties); 258 CFRelease(cv_pix_fmt); 259 CFRelease(buffer_attributes); 260 261 return status; 262} 263 264int ff_vda_destroy_decoder(struct vda_context *vda_ctx) 265{ 266 OSStatus status = kVDADecoderNoErr; 267 268 if (vda_ctx->decoder) 269 status = VDADecoderDestroy(vda_ctx->decoder); 270 271 return status; 272} 273 274static int vda_h264_uninit(AVCodecContext *avctx) 275{ 276 VDAContext *vda = avctx->internal->hwaccel_priv_data; 277 if (vda) { 278 av_freep(&vda->bitstream); 279 if (vda->frame) 280 CVPixelBufferRelease(vda->frame); 281 } 282 return 0; 283} 284 285AVHWAccel ff_h264_vda_old_hwaccel = { 286 .name = "h264_vda", 287 .type = AVMEDIA_TYPE_VIDEO, 288 .id = AV_CODEC_ID_H264, 289 .pix_fmt = AV_PIX_FMT_VDA_VLD, 290 .start_frame = vda_old_h264_start_frame, 291 .decode_slice = vda_old_h264_decode_slice, 292 .end_frame = vda_old_h264_end_frame, 293 .uninit = vda_h264_uninit, 294 .priv_data_size = sizeof(VDAContext), 295}; 296 297void ff_vda_output_callback(void *opaque, 298 CFDictionaryRef user_info, 299 OSStatus status, 300 uint32_t infoFlags, 301 CVImageBufferRef image_buffer) 302{ 303 AVCodecContext *ctx = opaque; 304 VDAContext *vda = ctx->internal->hwaccel_priv_data; 305 306 307 if (vda->frame) { 308 CVPixelBufferRelease(vda->frame); 309 vda->frame = NULL; 310 } 311 312 if (!image_buffer) 313 return; 314 315 vda->frame = CVPixelBufferRetain(image_buffer); 316} 317 318static int vda_h264_start_frame(AVCodecContext *avctx, 319 const uint8_t *buffer, 320 uint32_t size) 321{ 322 VDAContext *vda = avctx->internal->hwaccel_priv_data; 323 324 vda->bitstream_size = 0; 325 326 return 0; 327} 328 329static int vda_h264_decode_slice(AVCodecContext *avctx, 330 const uint8_t *buffer, 331 uint32_t size) 332{ 333 VDAContext *vda = avctx->internal->hwaccel_priv_data; 334 void *tmp; 335 336 tmp = av_fast_realloc(vda->bitstream, 337 &vda->allocated_size, 338 vda->bitstream_size + size + 4); 339 if (!tmp) 340 return AVERROR(ENOMEM); 341 342 vda->bitstream = tmp; 343 344 AV_WB32(vda->bitstream + vda->bitstream_size, size); 345 memcpy(vda->bitstream + vda->bitstream_size + 4, buffer, size); 346 347 vda->bitstream_size += size + 4; 348 349 return 0; 350} 351 352static void release_buffer(void *opaque, uint8_t *data) 353{ 354 CVImageBufferRef frame = (CVImageBufferRef)data; 355 CVPixelBufferRelease(frame); 356} 357 358static int vda_h264_end_frame(AVCodecContext *avctx) 359{ 360 H264Context *h = avctx->priv_data; 361 VDAContext *vda = avctx->internal->hwaccel_priv_data; 362 AVVDAContext *vda_ctx = avctx->hwaccel_context; 363 AVFrame *frame = &h->cur_pic_ptr->f; 364 uint32_t flush_flags = 1 << 0; ///< kVDADecoderFlush_emitFrames 365 CFDataRef coded_frame; 366 OSStatus status; 367 368 if (!vda->bitstream_size) 369 return AVERROR_INVALIDDATA; 370 371 372 coded_frame = CFDataCreate(kCFAllocatorDefault, 373 vda->bitstream, 374 vda->bitstream_size); 375 376 status = VDADecoderDecode(vda_ctx->decoder, 0, coded_frame, NULL); 377 378 if (status == kVDADecoderNoErr) 379 status = VDADecoderFlush(vda_ctx->decoder, flush_flags); 380 381 CFRelease(coded_frame); 382 383 if (status != kVDADecoderNoErr) { 384 av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status); 385 return AVERROR_UNKNOWN; 386 } 387 388 if (vda->frame) { 389 av_buffer_unref(&frame->buf[0]); 390 391 frame->buf[0] = av_buffer_create((uint8_t*)vda->frame, 392 sizeof(vda->frame), 393 release_buffer, NULL, 394 AV_BUFFER_FLAG_READONLY); 395 if (!frame->buf) 396 return AVERROR(ENOMEM); 397 398 frame->data[3] = (uint8_t*)vda->frame; 399 vda->frame = NULL; 400 } 401 402 return 0; 403} 404 405int ff_vda_default_init(AVCodecContext *avctx) 406{ 407 AVVDAContext *vda_ctx = avctx->hwaccel_context; 408 OSStatus status = kVDADecoderNoErr; 409 CFNumberRef height; 410 CFNumberRef width; 411 CFNumberRef format; 412 CFDataRef avc_data; 413 CFMutableDictionaryRef config_info; 414 CFMutableDictionaryRef buffer_attributes; 415 CFMutableDictionaryRef io_surface_properties; 416 CFNumberRef cv_pix_fmt; 417 int32_t fmt = 'avc1', pix_fmt = kCVPixelFormatType_422YpCbCr8; 418 419 // kCVPixelFormatType_420YpCbCr8Planar; 420 421 /* Each VCL NAL in the bitstream sent to the decoder 422 * is preceded by a 4 bytes length header. 423 * Change the avcC atom header if needed, to signal headers of 4 bytes. */ 424 if (avctx->extradata_size >= 4 && (avctx->extradata[4] & 0x03) != 0x03) { 425 uint8_t *rw_extradata; 426 427 if (!(rw_extradata = av_malloc(avctx->extradata_size))) 428 return AVERROR(ENOMEM); 429 430 memcpy(rw_extradata, avctx->extradata, avctx->extradata_size); 431 432 rw_extradata[4] |= 0x03; 433 434 avc_data = CFDataCreate(kCFAllocatorDefault, rw_extradata, avctx->extradata_size); 435 436 av_freep(&rw_extradata); 437 } else { 438 avc_data = CFDataCreate(kCFAllocatorDefault, 439 avctx->extradata, avctx->extradata_size); 440 } 441 442 config_info = CFDictionaryCreateMutable(kCFAllocatorDefault, 443 4, 444 &kCFTypeDictionaryKeyCallBacks, 445 &kCFTypeDictionaryValueCallBacks); 446 447 height = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &avctx->height); 448 width = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &avctx->width); 449 format = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &fmt); 450 CFDictionarySetValue(config_info, kVDADecoderConfiguration_Height, height); 451 CFDictionarySetValue(config_info, kVDADecoderConfiguration_Width, width); 452 CFDictionarySetValue(config_info, kVDADecoderConfiguration_avcCData, avc_data); 453 CFDictionarySetValue(config_info, kVDADecoderConfiguration_SourceFormat, format); 454 455 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault, 456 2, 457 &kCFTypeDictionaryKeyCallBacks, 458 &kCFTypeDictionaryValueCallBacks); 459 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault, 460 0, 461 &kCFTypeDictionaryKeyCallBacks, 462 &kCFTypeDictionaryValueCallBacks); 463 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, 464 kCFNumberSInt32Type, 465 &pix_fmt); 466 467 CFDictionarySetValue(buffer_attributes, 468 kCVPixelBufferPixelFormatTypeKey, 469 cv_pix_fmt); 470 CFDictionarySetValue(buffer_attributes, 471 kCVPixelBufferIOSurfacePropertiesKey, 472 io_surface_properties); 473 474 status = VDADecoderCreate(config_info, 475 buffer_attributes, 476 (VDADecoderOutputCallback *)ff_vda_output_callback, 477 avctx, 478 &vda_ctx->decoder); 479 480 CFRelease(format); 481 CFRelease(height); 482 CFRelease(width); 483 CFRelease(avc_data); 484 CFRelease(config_info); 485 CFRelease(cv_pix_fmt); 486 CFRelease(io_surface_properties); 487 CFRelease(buffer_attributes); 488 489 if (status != kVDADecoderNoErr) { 490 av_log(avctx, AV_LOG_ERROR, "Cannot initialize VDA %d\n", status); 491 } 492 493 switch (status) { 494 case kVDADecoderHardwareNotSupportedErr: 495 case kVDADecoderFormatNotSupportedErr: 496 return AVERROR(ENOSYS); 497 case kVDADecoderConfigurationError: 498 return AVERROR(EINVAL); 499 case kVDADecoderDecoderFailedErr: 500 return AVERROR_INVALIDDATA; 501 case kVDADecoderNoErr: 502 return 0; 503 default: 504 return AVERROR_UNKNOWN; 505 } 506} 507 508static int vda_h264_alloc_frame(AVCodecContext *avctx, AVFrame *frame) 509{ 510 frame->width = avctx->width; 511 frame->height = avctx->height; 512 frame->format = avctx->pix_fmt; 513 frame->buf[0] = av_buffer_alloc(1); 514 515 if (!frame->buf[0]) 516 return AVERROR(ENOMEM); 517 return 0; 518} 519 520AVHWAccel ff_h264_vda_hwaccel = { 521 .name = "h264_vda", 522 .type = AVMEDIA_TYPE_VIDEO, 523 .id = AV_CODEC_ID_H264, 524 .pix_fmt = AV_PIX_FMT_VDA, 525 .alloc_frame = vda_h264_alloc_frame, 526 .start_frame = vda_h264_start_frame, 527 .decode_slice = vda_h264_decode_slice, 528 .end_frame = vda_h264_end_frame, 529 .uninit = vda_h264_uninit, 530 .priv_data_size = sizeof(VDAContext), 531}; 532