1/* 2 * VMware Screen Codec (VMnc) decoder 3 * Copyright (c) 2006 Konstantin Shishkov 4 * 5 * This file is part of FFmpeg. 6 * 7 * FFmpeg is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * FFmpeg is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with FFmpeg; if not, write to the Free Software 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 */ 21 22/** 23 * @file 24 * VMware Screen Codec (VMnc) decoder 25 * As Alex Beregszaszi discovered, this is effectively RFB data dump 26 */ 27 28#include <stdio.h> 29#include <stdlib.h> 30 31#include "libavutil/common.h" 32#include "libavutil/intreadwrite.h" 33#include "avcodec.h" 34#include "internal.h" 35#include "bytestream.h" 36 37enum EncTypes { 38 MAGIC_WMVd = 0x574D5664, 39 MAGIC_WMVe, 40 MAGIC_WMVf, 41 MAGIC_WMVg, 42 MAGIC_WMVh, 43 MAGIC_WMVi, 44 MAGIC_WMVj 45}; 46 47enum HexTile_Flags { 48 HT_RAW = 1, // tile is raw 49 HT_BKG = 2, // background color is present 50 HT_FG = 4, // foreground color is present 51 HT_SUB = 8, // subrects are present 52 HT_CLR = 16 // each subrect has own color 53}; 54 55/* 56 * Decoder context 57 */ 58typedef struct VmncContext { 59 AVCodecContext *avctx; 60 AVFrame *pic; 61 62 int bpp; 63 int bpp2; 64 int bigendian; 65 uint8_t pal[768]; 66 int width, height; 67 GetByteContext gb; 68 69 /* cursor data */ 70 int cur_w, cur_h; 71 int cur_x, cur_y; 72 int cur_hx, cur_hy; 73 uint8_t *curbits, *curmask; 74 uint8_t *screendta; 75} VmncContext; 76 77/* read pixel value from stream */ 78static av_always_inline int vmnc_get_pixel(GetByteContext *gb, int bpp, int be) 79{ 80 switch (bpp * 2 + be) { 81 case 2: 82 case 3: 83 return bytestream2_get_byte(gb); 84 case 4: 85 return bytestream2_get_le16(gb); 86 case 5: 87 return bytestream2_get_be16(gb); 88 case 8: 89 return bytestream2_get_le32(gb); 90 case 9: 91 return bytestream2_get_be32(gb); 92 default: return 0; 93 } 94} 95 96static void load_cursor(VmncContext *c) 97{ 98 int i, j, p; 99 const int bpp = c->bpp2; 100 uint8_t *dst8 = c->curbits; 101 uint16_t *dst16 = (uint16_t *)c->curbits; 102 uint32_t *dst32 = (uint32_t *)c->curbits; 103 104 for (j = 0; j < c->cur_h; j++) { 105 for (i = 0; i < c->cur_w; i++) { 106 p = vmnc_get_pixel(&c->gb, bpp, c->bigendian); 107 if (bpp == 1) 108 *dst8++ = p; 109 if (bpp == 2) 110 *dst16++ = p; 111 if (bpp == 4) 112 *dst32++ = p; 113 } 114 } 115 dst8 = c->curmask; 116 dst16 = (uint16_t*)c->curmask; 117 dst32 = (uint32_t*)c->curmask; 118 for (j = 0; j < c->cur_h; j++) { 119 for (i = 0; i < c->cur_w; i++) { 120 p = vmnc_get_pixel(&c->gb, bpp, c->bigendian); 121 if (bpp == 1) 122 *dst8++ = p; 123 if (bpp == 2) 124 *dst16++ = p; 125 if (bpp == 4) 126 *dst32++ = p; 127 } 128 } 129} 130 131static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy) 132{ 133 int i, j; 134 int w, h, x, y; 135 w = c->cur_w; 136 if (c->width < c->cur_x + c->cur_w) 137 w = c->width - c->cur_x; 138 h = c->cur_h; 139 if (c->height < c->cur_y + c->cur_h) 140 h = c->height - c->cur_y; 141 x = c->cur_x; 142 y = c->cur_y; 143 if (x < 0) { 144 w += x; 145 x = 0; 146 } 147 if (y < 0) { 148 h += y; 149 y = 0; 150 } 151 152 if ((w < 1) || (h < 1)) 153 return; 154 dst += x * c->bpp2 + y * stride; 155 156 if (c->bpp2 == 1) { 157 uint8_t *cd = c->curbits, *msk = c->curmask; 158 for (j = 0; j < h; j++) { 159 for (i = 0; i < w; i++) 160 dst[i] = (dst[i] & cd[i]) ^ msk[i]; 161 msk += c->cur_w; 162 cd += c->cur_w; 163 dst += stride; 164 } 165 } else if (c->bpp2 == 2) { 166 uint16_t *cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask; 167 uint16_t *dst2; 168 for (j = 0; j < h; j++) { 169 dst2 = (uint16_t*)dst; 170 for (i = 0; i < w; i++) 171 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; 172 msk += c->cur_w; 173 cd += c->cur_w; 174 dst += stride; 175 } 176 } else if (c->bpp2 == 4) { 177 uint32_t *cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask; 178 uint32_t *dst2; 179 for (j = 0; j < h; j++) { 180 dst2 = (uint32_t*)dst; 181 for (i = 0; i < w; i++) 182 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; 183 msk += c->cur_w; 184 cd += c->cur_w; 185 dst += stride; 186 } 187 } 188} 189 190/* fill rectangle with given color */ 191static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy, 192 int w, int h, int color, 193 int bpp, int stride) 194{ 195 int i, j; 196 dst += dx * bpp + dy * stride; 197 if (bpp == 1) { 198 for (j = 0; j < h; j++) { 199 memset(dst, color, w); 200 dst += stride; 201 } 202 } else if (bpp == 2) { 203 uint16_t *dst2; 204 for (j = 0; j < h; j++) { 205 dst2 = (uint16_t*)dst; 206 for (i = 0; i < w; i++) 207 *dst2++ = color; 208 dst += stride; 209 } 210 } else if (bpp == 4) { 211 uint32_t *dst2; 212 for (j = 0; j < h; j++) { 213 dst2 = (uint32_t*)dst; 214 for (i = 0; i < w; i++) 215 dst2[i] = color; 216 dst += stride; 217 } 218 } 219} 220 221static av_always_inline void paint_raw(uint8_t *dst, int w, int h, 222 GetByteContext *gb, int bpp, 223 int be, int stride) 224{ 225 int i, j, p; 226 for (j = 0; j < h; j++) { 227 for (i = 0; i < w; i++) { 228 p = vmnc_get_pixel(gb, bpp, be); 229 switch (bpp) { 230 case 1: 231 dst[i] = p; 232 break; 233 case 2: 234 ((uint16_t*)dst)[i] = p; 235 break; 236 case 4: 237 ((uint32_t*)dst)[i] = p; 238 break; 239 } 240 } 241 dst += stride; 242 } 243} 244 245static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb, 246 int w, int h, int stride) 247{ 248 int i, j, k; 249 int bg = 0, fg = 0, rects, color, flags, xy, wh; 250 const int bpp = c->bpp2; 251 uint8_t *dst2; 252 int bw = 16, bh = 16; 253 254 for (j = 0; j < h; j += 16) { 255 dst2 = dst; 256 bw = 16; 257 if (j + 16 > h) 258 bh = h - j; 259 for (i = 0; i < w; i += 16, dst2 += 16 * bpp) { 260 if (bytestream2_get_bytes_left(gb) <= 0) { 261 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); 262 return AVERROR_INVALIDDATA; 263 } 264 if (i + 16 > w) 265 bw = w - i; 266 flags = bytestream2_get_byte(gb); 267 if (flags & HT_RAW) { 268 if (bytestream2_get_bytes_left(gb) < bw * bh * bpp) { 269 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); 270 return AVERROR_INVALIDDATA; 271 } 272 paint_raw(dst2, bw, bh, gb, bpp, c->bigendian, stride); 273 } else { 274 if (flags & HT_BKG) 275 bg = vmnc_get_pixel(gb, bpp, c->bigendian); 276 if (flags & HT_FG) 277 fg = vmnc_get_pixel(gb, bpp, c->bigendian); 278 rects = 0; 279 if (flags & HT_SUB) 280 rects = bytestream2_get_byte(gb); 281 color = !!(flags & HT_CLR); 282 283 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride); 284 285 if (bytestream2_get_bytes_left(gb) < rects * (color * bpp + 2)) { 286 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); 287 return AVERROR_INVALIDDATA; 288 } 289 for (k = 0; k < rects; k++) { 290 if (color) 291 fg = vmnc_get_pixel(gb, bpp, c->bigendian); 292 xy = bytestream2_get_byte(gb); 293 wh = bytestream2_get_byte(gb); 294 if ( (xy >> 4) + (wh >> 4) + 1 > w - i 295 || (xy & 0xF) + (wh & 0xF)+1 > h - j) { 296 av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n"); 297 return AVERROR_INVALIDDATA; 298 } 299 paint_rect(dst2, xy >> 4, xy & 0xF, 300 (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride); 301 } 302 } 303 } 304 dst += stride * 16; 305 } 306 return 0; 307} 308 309static void reset_buffers(VmncContext *c) 310{ 311 av_freep(&c->curbits); 312 av_freep(&c->curmask); 313 av_freep(&c->screendta); 314 c->cur_w = c->cur_h = 0; 315 c->cur_hx = c->cur_hy = 0; 316 317} 318 319static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, 320 AVPacket *avpkt) 321{ 322 const uint8_t *buf = avpkt->data; 323 int buf_size = avpkt->size; 324 VmncContext * const c = avctx->priv_data; 325 GetByteContext *gb = &c->gb; 326 uint8_t *outptr; 327 int dx, dy, w, h, depth, enc, chunks, res, size_left, ret; 328 329 if ((ret = ff_reget_buffer(avctx, c->pic)) < 0) 330 return ret; 331 332 bytestream2_init(gb, buf, buf_size); 333 334 c->pic->key_frame = 0; 335 c->pic->pict_type = AV_PICTURE_TYPE_P; 336 337 // restore screen after cursor 338 if (c->screendta) { 339 int i; 340 w = c->cur_w; 341 if (c->width < c->cur_x + w) 342 w = c->width - c->cur_x; 343 h = c->cur_h; 344 if (c->height < c->cur_y + h) 345 h = c->height - c->cur_y; 346 dx = c->cur_x; 347 if (dx < 0) { 348 w += dx; 349 dx = 0; 350 } 351 dy = c->cur_y; 352 if (dy < 0) { 353 h += dy; 354 dy = 0; 355 } 356 if ((w > 0) && (h > 0)) { 357 outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; 358 for (i = 0; i < h; i++) { 359 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, 360 w * c->bpp2); 361 outptr += c->pic->linesize[0]; 362 } 363 } 364 } 365 bytestream2_skip(gb, 2); 366 chunks = bytestream2_get_be16(gb); 367 while (chunks--) { 368 if (bytestream2_get_bytes_left(gb) < 12) { 369 av_log(avctx, AV_LOG_ERROR, "Premature end of data!\n"); 370 return -1; 371 } 372 dx = bytestream2_get_be16(gb); 373 dy = bytestream2_get_be16(gb); 374 w = bytestream2_get_be16(gb); 375 h = bytestream2_get_be16(gb); 376 enc = bytestream2_get_be32(gb); 377 outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; 378 size_left = bytestream2_get_bytes_left(gb); 379 switch (enc) { 380 case MAGIC_WMVd: // cursor 381 if (w*(int64_t)h*c->bpp2 > INT_MAX/2 - 2) { 382 av_log(avctx, AV_LOG_ERROR, "dimensions too large\n"); 383 return AVERROR_INVALIDDATA; 384 } 385 if (size_left < 2 + w * h * c->bpp2 * 2) { 386 av_log(avctx, AV_LOG_ERROR, 387 "Premature end of data! (need %i got %i)\n", 388 2 + w * h * c->bpp2 * 2, size_left); 389 return AVERROR_INVALIDDATA; 390 } 391 bytestream2_skip(gb, 2); 392 c->cur_w = w; 393 c->cur_h = h; 394 c->cur_hx = dx; 395 c->cur_hy = dy; 396 if ((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) { 397 av_log(avctx, AV_LOG_ERROR, 398 "Cursor hot spot is not in image: " 399 "%ix%i of %ix%i cursor size\n", 400 c->cur_hx, c->cur_hy, c->cur_w, c->cur_h); 401 c->cur_hx = c->cur_hy = 0; 402 } 403 if (c->cur_w * c->cur_h >= INT_MAX / c->bpp2) { 404 reset_buffers(c); 405 return AVERROR(EINVAL); 406 } else { 407 int screen_size = c->cur_w * c->cur_h * c->bpp2; 408 if ((ret = av_reallocp(&c->curbits, screen_size)) < 0 || 409 (ret = av_reallocp(&c->curmask, screen_size)) < 0 || 410 (ret = av_reallocp(&c->screendta, screen_size)) < 0) { 411 reset_buffers(c); 412 return ret; 413 } 414 } 415 load_cursor(c); 416 break; 417 case MAGIC_WMVe: // unknown 418 bytestream2_skip(gb, 2); 419 break; 420 case MAGIC_WMVf: // update cursor position 421 c->cur_x = dx - c->cur_hx; 422 c->cur_y = dy - c->cur_hy; 423 break; 424 case MAGIC_WMVg: // unknown 425 bytestream2_skip(gb, 10); 426 break; 427 case MAGIC_WMVh: // unknown 428 bytestream2_skip(gb, 4); 429 break; 430 case MAGIC_WMVi: // ServerInitialization struct 431 c->pic->key_frame = 1; 432 c->pic->pict_type = AV_PICTURE_TYPE_I; 433 depth = bytestream2_get_byte(gb); 434 if (depth != c->bpp) { 435 av_log(avctx, AV_LOG_INFO, 436 "Depth mismatch. Container %i bpp, " 437 "Frame data: %i bpp\n", 438 c->bpp, depth); 439 } 440 bytestream2_skip(gb, 1); 441 c->bigendian = bytestream2_get_byte(gb); 442 if (c->bigendian & (~1)) { 443 av_log(avctx, AV_LOG_INFO, 444 "Invalid header: bigendian flag = %i\n", c->bigendian); 445 return AVERROR_INVALIDDATA; 446 } 447 //skip the rest of pixel format data 448 bytestream2_skip(gb, 13); 449 break; 450 case MAGIC_WMVj: // unknown 451 bytestream2_skip(gb, 2); 452 break; 453 case 0x00000000: // raw rectangle data 454 if ((dx + w > c->width) || (dy + h > c->height)) { 455 av_log(avctx, AV_LOG_ERROR, 456 "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", 457 w, h, dx, dy, c->width, c->height); 458 return AVERROR_INVALIDDATA; 459 } 460 if (size_left < w * h * c->bpp2) { 461 av_log(avctx, AV_LOG_ERROR, 462 "Premature end of data! (need %i got %i)\n", 463 w * h * c->bpp2, size_left); 464 return AVERROR_INVALIDDATA; 465 } 466 paint_raw(outptr, w, h, gb, c->bpp2, c->bigendian, 467 c->pic->linesize[0]); 468 break; 469 case 0x00000005: // HexTile encoded rectangle 470 if ((dx + w > c->width) || (dy + h > c->height)) { 471 av_log(avctx, AV_LOG_ERROR, 472 "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", 473 w, h, dx, dy, c->width, c->height); 474 return AVERROR_INVALIDDATA; 475 } 476 res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]); 477 if (res < 0) 478 return res; 479 break; 480 default: 481 av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc); 482 chunks = 0; // leave chunks decoding loop 483 } 484 } 485 if (c->screendta) { 486 int i; 487 // save screen data before painting cursor 488 w = c->cur_w; 489 if (c->width < c->cur_x + w) 490 w = c->width - c->cur_x; 491 h = c->cur_h; 492 if (c->height < c->cur_y + h) 493 h = c->height - c->cur_y; 494 dx = c->cur_x; 495 if (dx < 0) { 496 w += dx; 497 dx = 0; 498 } 499 dy = c->cur_y; 500 if (dy < 0) { 501 h += dy; 502 dy = 0; 503 } 504 if ((w > 0) && (h > 0)) { 505 outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; 506 for (i = 0; i < h; i++) { 507 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, 508 w * c->bpp2); 509 outptr += c->pic->linesize[0]; 510 } 511 outptr = c->pic->data[0]; 512 put_cursor(outptr, c->pic->linesize[0], c, c->cur_x, c->cur_y); 513 } 514 } 515 *got_frame = 1; 516 if ((ret = av_frame_ref(data, c->pic)) < 0) 517 return ret; 518 519 /* always report that the buffer was completely consumed */ 520 return buf_size; 521} 522 523static av_cold int decode_init(AVCodecContext *avctx) 524{ 525 VmncContext * const c = avctx->priv_data; 526 527 c->avctx = avctx; 528 c->width = avctx->width; 529 c->height = avctx->height; 530 c->bpp = avctx->bits_per_coded_sample; 531 c->bpp2 = c->bpp / 8; 532 533 switch (c->bpp) { 534 case 8: 535 avctx->pix_fmt = AV_PIX_FMT_PAL8; 536 break; 537 case 16: 538 avctx->pix_fmt = AV_PIX_FMT_RGB555; 539 break; 540 case 32: 541 avctx->pix_fmt = AV_PIX_FMT_RGB32; 542 break; 543 default: 544 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); 545 return AVERROR_INVALIDDATA; 546 } 547 548 c->pic = av_frame_alloc(); 549 if (!c->pic) 550 return AVERROR(ENOMEM); 551 552 return 0; 553} 554 555static av_cold int decode_end(AVCodecContext *avctx) 556{ 557 VmncContext * const c = avctx->priv_data; 558 559 av_frame_free(&c->pic); 560 561 av_freep(&c->curbits); 562 av_freep(&c->curmask); 563 av_freep(&c->screendta); 564 return 0; 565} 566 567AVCodec ff_vmnc_decoder = { 568 .name = "vmnc", 569 .long_name = NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"), 570 .type = AVMEDIA_TYPE_VIDEO, 571 .id = AV_CODEC_ID_VMNC, 572 .priv_data_size = sizeof(VmncContext), 573 .init = decode_init, 574 .close = decode_end, 575 .decode = decode_frame, 576 .capabilities = CODEC_CAP_DR1, 577}; 578