src/gallium/drivers/radeon/radeon_vcn_enc.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2017 Advanced Micro Devices, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include <stdio.h>
  29
  30 #include "pipe/p_video_codec.h"
  31
  32 #include "util/u_video.h"
  33 #include "util/u_memory.h"
  34
  35 #include "vl/vl_video_buffer.h"
  36
  37 #include "radeonsi/si_pipe.h"
  38 #include "radeon_video.h"
  39 #include "radeon_vcn_enc.h"
  40
  41 static const unsigned index_to_shifts[4] = {24, 16, 8, 0};
  42
  43 static void radeon_vcn_enc_get_param(struct radeon_encoder *enc, struct pipe_picture_desc *picture)
  44 {
  45    if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_MPEG4_AVC) {
  46       struct pipe_h264_enc_picture_desc *pic = (struct pipe_h264_enc_picture_desc *)picture;
  47       enc->enc_pic.picture_type = pic->picture_type;
  48       enc->enc_pic.frame_num = pic->frame_num;
  49       enc->enc_pic.pic_order_cnt = pic->pic_order_cnt;
  50       enc->enc_pic.pic_order_cnt_type = pic->pic_order_cnt_type;
  51       enc->enc_pic.ref_idx_l0 = pic->ref_idx_l0;
  52       enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1;
  53       enc->enc_pic.not_referenced = pic->not_referenced;
  54       enc->enc_pic.is_idr = (pic->picture_type == PIPE_H264_ENC_PICTURE_TYPE_IDR);
  55       enc->enc_pic.crop_left = 0;
  56       enc->enc_pic.crop_right = (align(enc->base.width, 16) - enc->base.width) / 2;
  57       enc->enc_pic.crop_top = 0;
  58       enc->enc_pic.crop_bottom = (align(enc->base.height, 16) - enc->base.height) / 2;
  59       enc->enc_pic.rc_layer_init.target_bit_rate = pic->rate_ctrl.target_bitrate;
  60       enc->enc_pic.rc_layer_init.peak_bit_rate = pic->rate_ctrl.peak_bitrate;
  61       enc->enc_pic.rc_layer_init.frame_rate_num = pic->rate_ctrl.frame_rate_num;
  62       enc->enc_pic.rc_layer_init.frame_rate_den = pic->rate_ctrl.frame_rate_den;
  63       enc->enc_pic.rc_layer_init.vbv_buffer_size = pic->rate_ctrl.vbv_buffer_size;
  64       enc->enc_pic.rc_layer_init.avg_target_bits_per_picture = pic->rate_ctrl.target_bits_picture;
  65       enc->enc_pic.rc_layer_init.peak_bits_per_picture_integer = pic->rate_ctrl.peak_bits_picture_integer;
  66       enc->enc_pic.rc_layer_init.peak_bits_per_picture_fractional = pic->rate_ctrl.peak_bits_picture_fraction;
  67       enc->enc_pic.rc_session_init.vbv_buffer_level = pic->rate_ctrl.vbv_buf_lv;
  68       enc->enc_pic.rc_per_pic.qp = pic->quant_i_frames;
  69       enc->enc_pic.rc_per_pic.min_qp_app = 0;
  70       enc->enc_pic.rc_per_pic.max_qp_app = 51;
  71       enc->enc_pic.rc_per_pic.max_au_size = 0;
  72       enc->enc_pic.rc_per_pic.enabled_filler_data = pic->rate_ctrl.fill_data_enable;
  73       enc->enc_pic.rc_per_pic.skip_frame_enable = false;
  74       enc->enc_pic.rc_per_pic.enforce_hrd = pic->rate_ctrl.enforce_hrd;
  75       switch(pic->rate_ctrl.rate_ctrl_method) {
  76          case PIPE_H264_ENC_RATE_CONTROL_METHOD_DISABLE:
  77             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
  78             break;
  79          case PIPE_H264_ENC_RATE_CONTROL_METHOD_CONSTANT_SKIP:
  80          case PIPE_H264_ENC_RATE_CONTROL_METHOD_CONSTANT:
  81             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_CBR;
  82             break;
  83          case PIPE_H264_ENC_RATE_CONTROL_METHOD_VARIABLE_SKIP:
  84          case PIPE_H264_ENC_RATE_CONTROL_METHOD_VARIABLE:
  85             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_PEAK_CONSTRAINED_VBR;
  86             break;
  87          default:
  88             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
  89       }
  90    } else if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_HEVC) {
  91       struct pipe_h265_enc_picture_desc *pic = (struct pipe_h265_enc_picture_desc *)picture;
  92       enc->enc_pic.picture_type = pic->picture_type;
  93       enc->enc_pic.frame_num = pic->frame_num;
  94       enc->enc_pic.pic_order_cnt = pic->pic_order_cnt;
  95       enc->enc_pic.pic_order_cnt_type = pic->pic_order_cnt_type;
  96       enc->enc_pic.ref_idx_l0 = pic->ref_idx_l0;
  97       enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1;
  98       enc->enc_pic.not_referenced = pic->not_referenced;
  99       enc->enc_pic.is_idr = (pic->picture_type == PIPE_H265_ENC_PICTURE_TYPE_IDR) ||
 100                             (pic->picture_type == PIPE_H265_ENC_PICTURE_TYPE_I);
 101       enc->enc_pic.crop_left = 0;
 102       enc->enc_pic.crop_right = (align(enc->base.width, 16) - enc->base.width) / 2;
 103       enc->enc_pic.crop_top = 0;
 104       enc->enc_pic.crop_bottom = (align(enc->base.height, 16) - enc->base.height) / 2;
 105       enc->enc_pic.general_tier_flag = pic->seq.general_tier_flag;
 106       enc->enc_pic.general_profile_idc = pic->seq.general_profile_idc;
 107       enc->enc_pic.general_level_idc = pic->seq.general_level_idc;
 108       enc->enc_pic.max_poc = pic->seq.intra_period;
 109       enc->enc_pic.log2_max_poc = 0;
 110       for (int i = enc->enc_pic.max_poc; i != 0; enc->enc_pic.log2_max_poc++)
 111          i = (i >> 1);
 112       enc->enc_pic.chroma_format_idc = pic->seq.chroma_format_idc;
 113       enc->enc_pic.pic_width_in_luma_samples = pic->seq.pic_width_in_luma_samples;
 114       enc->enc_pic.pic_height_in_luma_samples = pic->seq.pic_height_in_luma_samples;
 115       enc->enc_pic.log2_diff_max_min_luma_coding_block_size = pic->seq.log2_diff_max_min_luma_coding_block_size;
 116       enc->enc_pic.log2_min_transform_block_size_minus2 = pic->seq.log2_min_transform_block_size_minus2;
 117       enc->enc_pic.log2_diff_max_min_transform_block_size = pic->seq.log2_diff_max_min_transform_block_size;
 118       enc->enc_pic.max_transform_hierarchy_depth_inter = pic->seq.max_transform_hierarchy_depth_inter;
 119       enc->enc_pic.max_transform_hierarchy_depth_intra = pic->seq.max_transform_hierarchy_depth_intra;
 120       enc->enc_pic.log2_parallel_merge_level_minus2 = pic->pic.log2_parallel_merge_level_minus2;
 121       enc->enc_pic.bit_depth_luma_minus8 = pic->seq.bit_depth_luma_minus8;
 122       enc->enc_pic.bit_depth_chroma_minus8 = pic->seq.bit_depth_chroma_minus8;
 123       enc->enc_pic.nal_unit_type = pic->pic.nal_unit_type;
 124       enc->enc_pic.max_num_merge_cand = pic->slice.max_num_merge_cand;
 125       enc->enc_pic.sample_adaptive_offset_enabled_flag = pic->seq.sample_adaptive_offset_enabled_flag;
 126       enc->enc_pic.pcm_enabled_flag = pic->seq.pcm_enabled_flag;
 127       enc->enc_pic.sps_temporal_mvp_enabled_flag = pic->seq.sps_temporal_mvp_enabled_flag;
 128       enc->enc_pic.hevc_deblock.loop_filter_across_slices_enabled = pic->slice.slice_loop_filter_across_slices_enabled_flag;
 129       enc->enc_pic.hevc_deblock.deblocking_filter_disabled = pic->slice.slice_deblocking_filter_disabled_flag;
 130       enc->enc_pic.hevc_deblock.beta_offset_div2 = pic->slice.slice_beta_offset_div2;
 131       enc->enc_pic.hevc_deblock.tc_offset_div2 = pic->slice.slice_tc_offset_div2;
 132       enc->enc_pic.hevc_deblock.cb_qp_offset = pic->slice.slice_cb_qp_offset;
 133       enc->enc_pic.hevc_deblock.cr_qp_offset = pic->slice.slice_cr_qp_offset;
 134       enc->enc_pic.hevc_spec_misc.log2_min_luma_coding_block_size_minus3 = pic->seq.log2_min_luma_coding_block_size_minus3;
 135       enc->enc_pic.hevc_spec_misc.amp_disabled = !pic->seq.amp_enabled_flag;
 136       enc->enc_pic.hevc_spec_misc.strong_intra_smoothing_enabled = pic->seq.strong_intra_smoothing_enabled_flag;
 137       enc->enc_pic.hevc_spec_misc.constrained_intra_pred_flag = pic->pic.constrained_intra_pred_flag;
 138       enc->enc_pic.hevc_spec_misc.cabac_init_flag = pic->slice.cabac_init_flag;
 139       enc->enc_pic.hevc_spec_misc.half_pel_enabled = 1;
 140       enc->enc_pic.hevc_spec_misc.quarter_pel_enabled = 1;
 141       enc->enc_pic.rc_layer_init.target_bit_rate = pic->rc.target_bitrate;
 142       enc->enc_pic.rc_layer_init.peak_bit_rate = pic->rc.peak_bitrate;
 143       enc->enc_pic.rc_layer_init.frame_rate_num = pic->rc.frame_rate_num;
 144       enc->enc_pic.rc_layer_init.frame_rate_den = pic->rc.frame_rate_den;
 145       enc->enc_pic.rc_layer_init.vbv_buffer_size = pic->rc.vbv_buffer_size;
 146       enc->enc_pic.rc_layer_init.avg_target_bits_per_picture = pic->rc.target_bits_picture;
 147       enc->enc_pic.rc_layer_init.peak_bits_per_picture_integer = pic->rc.peak_bits_picture_integer;
 148       enc->enc_pic.rc_layer_init.peak_bits_per_picture_fractional = pic->rc.peak_bits_picture_fraction;
 149       enc->enc_pic.rc_session_init.vbv_buffer_level = pic->rc.vbv_buf_lv;
 150       enc->enc_pic.rc_per_pic.qp = pic->rc.quant_i_frames;
 151       enc->enc_pic.rc_per_pic.min_qp_app = 0;
 152       enc->enc_pic.rc_per_pic.max_qp_app = 51;
 153       enc->enc_pic.rc_per_pic.max_au_size = 0;
 154       enc->enc_pic.rc_per_pic.enabled_filler_data = pic->rc.fill_data_enable;
 155       enc->enc_pic.rc_per_pic.skip_frame_enable = false;
 156       enc->enc_pic.rc_per_pic.enforce_hrd = pic->rc.enforce_hrd;
 157       switch(pic->rc.rate_ctrl_method) {
 158          case PIPE_H265_ENC_RATE_CONTROL_METHOD_DISABLE:
 159          enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
 160             break;
 161          case PIPE_H265_ENC_RATE_CONTROL_METHOD_CONSTANT_SKIP:
 162          case PIPE_H265_ENC_RATE_CONTROL_METHOD_CONSTANT:
 163             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_CBR;
 164             break;
 165          case PIPE_H265_ENC_RATE_CONTROL_METHOD_VARIABLE_SKIP:
 166          case PIPE_H265_ENC_RATE_CONTROL_METHOD_VARIABLE:
 167             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_PEAK_CONSTRAINED_VBR;
 168             break;
 169          default:
 170             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
 171       }
 172    }
 173 }
 174
 175 static void flush(struct radeon_encoder *enc)
 176 {
 177         enc->ws->cs_flush(enc->cs, PIPE_FLUSH_ASYNC, NULL);
 178 }
 179
 180 static void radeon_enc_flush(struct pipe_video_codec *encoder)
 181 {
 182         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 183         flush(enc);
 184 }
 185
 186 static void radeon_enc_cs_flush(void *ctx, unsigned flags,
 187                                                                 struct pipe_fence_handle **fence)
 188 {
 189         // just ignored
 190 }
 191
 192 static unsigned get_cpb_num(struct radeon_encoder *enc)
 193 {
 194         unsigned w = align(enc->base.width, 16) / 16;
 195         unsigned h = align(enc->base.height, 16) / 16;
 196         unsigned dpb;
 197
 198         switch (enc->base.level) {
 199         case 10:
 200                 dpb = 396;
 201                 break;
 202         case 11:
 203                 dpb = 900;
 204                 break;
 205         case 12:
 206         case 13:
 207         case 20:
 208                 dpb = 2376;
 209                 break;
 210         case 21:
 211                 dpb = 4752;
 212                 break;
 213         case 22:
 214         case 30:
 215                 dpb = 8100;
 216                 break;
 217         case 31:
 218                 dpb = 18000;
 219                 break;
 220         case 32:
 221                 dpb = 20480;
 222                 break;
 223         case 40:
 224         case 41:
 225                 dpb = 32768;
 226                 break;
 227         case 42:
 228                 dpb = 34816;
 229                 break;
 230         case 50:
 231                 dpb = 110400;
 232                 break;
 233         default:
 234         case 51:
 235         case 52:
 236                 dpb = 184320;
 237                 break;
 238         }
 239
 240         return MIN2(dpb / (w * h), 16);
 241 }
 242
 243 static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
 244                                                          struct pipe_video_buffer *source,
 245                                                          struct pipe_picture_desc *picture)
 246 {
 247         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 248         struct vl_video_buffer *vid_buf = (struct vl_video_buffer *)source;
 249
 250         radeon_vcn_enc_get_param(enc, picture);
 251
 252         enc->get_buffer(vid_buf->resources[0], &enc->handle, &enc->luma);
 253         enc->get_buffer(vid_buf->resources[1], NULL, &enc->chroma);
 254
 255         enc->need_feedback = false;
 256
 257         if (!enc->stream_handle) {
 258                 struct rvid_buffer fb;
 259                 enc->stream_handle = si_vid_alloc_stream_handle();
 260                 enc->si = CALLOC_STRUCT(rvid_buffer);
 261                 si_vid_create_buffer(enc->screen, enc->si, 128 * 1024, PIPE_USAGE_STAGING);
 262                 si_vid_create_buffer(enc->screen, &fb, 4096, PIPE_USAGE_STAGING);
 263                 enc->fb = &fb;
 264                 enc->begin(enc);
 265                 flush(enc);
 266                 si_vid_destroy_buffer(&fb);
 267         }
 268 }
 269
 270 static void radeon_enc_encode_bitstream(struct pipe_video_codec *encoder,
 271                                                                   struct pipe_video_buffer *source,
 272                                                                   struct pipe_resource *destination,
 273                                                                   void **fb)
 274 {
 275         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 276         enc->get_buffer(destination, &enc->bs_handle, NULL);
 277         enc->bs_size = destination->width0;
 278
 279         *fb = enc->fb = CALLOC_STRUCT(rvid_buffer);
 280
 281         if (!si_vid_create_buffer(enc->screen, enc->fb, 4096, PIPE_USAGE_STAGING)) {
 282                 RVID_ERR("Can't create feedback buffer.\n");
 283                 return;
 284         }
 285
 286         enc->need_feedback = true;
 287         enc->encode(enc);
 288 }
 289
 290 static void radeon_enc_end_frame(struct pipe_video_codec *encoder,
 291                                                    struct pipe_video_buffer *source,
 292                                                    struct pipe_picture_desc *picture)
 293 {
 294         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 295         flush(enc);
 296 }
 297
 298 static void radeon_enc_destroy(struct pipe_video_codec *encoder)
 299 {
 300         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 301
 302         if (enc->stream_handle) {
 303                 struct rvid_buffer fb;
 304                 enc->need_feedback = false;
 305                 si_vid_create_buffer(enc->screen, &fb, 512, PIPE_USAGE_STAGING);
 306                 enc->fb = &fb;
 307                 enc->destroy(enc);
 308                 flush(enc);
 309                 si_vid_destroy_buffer(&fb);
 310         }
 311
 312         si_vid_destroy_buffer(&enc->cpb);
 313         enc->ws->cs_destroy(enc->cs);
 314         FREE(enc);
 315 }
 316
 317 static void radeon_enc_get_feedback(struct pipe_video_codec *encoder,
 318                                                           void *feedback, unsigned *size)
 319 {
 320         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 321         struct rvid_buffer *fb = feedback;
 322
 323         if (size) {
 324                 uint32_t *ptr = enc->ws->buffer_map(
 325                         fb->res->buf, enc->cs,
 326                         PIPE_TRANSFER_READ_WRITE | RADEON_TRANSFER_TEMPORARY);
 327                 if (ptr[1])
 328                         *size = ptr[6];
 329                 else
 330                         *size = 0;
 331                 enc->ws->buffer_unmap(fb->res->buf);
 332         }
 333
 334         si_vid_destroy_buffer(fb);
 335         FREE(fb);
 336 }
 337
 338 struct pipe_video_codec *radeon_create_encoder(struct pipe_context *context,
 339                 const struct pipe_video_codec *templ,
 340                 struct radeon_winsys* ws,
 341                 radeon_enc_get_buffer get_buffer)
 342 {
 343         struct si_screen *sscreen = (struct si_screen *)context->screen;
 344         struct si_context *sctx = (struct si_context*)context;
 345         struct radeon_encoder *enc;
 346         struct pipe_video_buffer *tmp_buf, templat = {};
 347         struct radeon_surf *tmp_surf;
 348         unsigned cpb_size;
 349
 350         enc = CALLOC_STRUCT(radeon_encoder);
 351
 352         if (!enc)
 353                 return NULL;
 354
 355         enc->alignment = 256;
 356         enc->base = *templ;
 357         enc->base.context = context;
 358         enc->base.destroy = radeon_enc_destroy;
 359         enc->base.begin_frame = radeon_enc_begin_frame;
 360         enc->base.encode_bitstream = radeon_enc_encode_bitstream;
 361         enc->base.end_frame = radeon_enc_end_frame;
 362         enc->base.flush = radeon_enc_flush;
 363         enc->base.get_feedback = radeon_enc_get_feedback;
 364         enc->get_buffer = get_buffer;
 365         enc->bits_in_shifter = 0;
 366         enc->screen = context->screen;
 367         enc->ws = ws;
 368         enc->cs = ws->cs_create(sctx->ctx, RING_VCN_ENC, radeon_enc_cs_flush,
 369                                 enc, false);
 370
 371         if (!enc->cs) {
 372                 RVID_ERR("Can't get command submission context.\n");
 373                 goto error;
 374         }
 375
 376         struct rvid_buffer si;
 377         si_vid_create_buffer(enc->screen, &si, 128 * 1024, PIPE_USAGE_STAGING);
 378         enc->si = &si;
 379
 380         templat.buffer_format = PIPE_FORMAT_NV12;
 381         templat.chroma_format = PIPE_VIDEO_CHROMA_FORMAT_420;
 382         templat.width = enc->base.width;
 383         templat.height = enc->base.height;
 384         templat.interlaced = false;
 385
 386         if (!(tmp_buf = context->create_video_buffer(context, &templat))) {
 387                 RVID_ERR("Can't create video buffer.\n");
 388                 goto error;
 389         }
 390
 391         enc->cpb_num = get_cpb_num(enc);
 392
 393         if (!enc->cpb_num)
 394                 goto error;
 395
 396         get_buffer(((struct vl_video_buffer *)tmp_buf)->resources[0], NULL, &tmp_surf);
 397
 398         cpb_size = (sscreen->info.chip_class < GFX9) ?
 399                            align(tmp_surf->u.legacy.level[0].nblk_x * tmp_surf->bpe, 128) *
 400                            align(tmp_surf->u.legacy.level[0].nblk_y, 32) :
 401                            align(tmp_surf->u.gfx9.surf_pitch * tmp_surf->bpe, 256) *
 402                            align(tmp_surf->u.gfx9.surf_height, 32);
 403
 404         cpb_size = cpb_size * 3 / 2;
 405         cpb_size = cpb_size * enc->cpb_num;
 406         tmp_buf->destroy(tmp_buf);
 407
 408         if (!si_vid_create_buffer(enc->screen, &enc->cpb, cpb_size, PIPE_USAGE_DEFAULT)) {
 409                 RVID_ERR("Can't create CPB buffer.\n");
 410                 goto error;
 411         }
 412
 413         radeon_enc_1_2_init(enc);
 414
 415         return &enc->base;
 416
 417 error:
 418         if (enc->cs)
 419                 enc->ws->cs_destroy(enc->cs);
 420
 421         si_vid_destroy_buffer(&enc->cpb);
 422
 423         FREE(enc);
 424         return NULL;
 425 }
 426
 427 void radeon_enc_add_buffer(struct radeon_encoder *enc, struct pb_buffer *buf,
 428                            enum radeon_bo_usage usage, enum radeon_bo_domain domain,
 429                            signed offset)
 430 {
 431         enc->ws->cs_add_buffer(enc->cs, buf, usage | RADEON_USAGE_SYNCHRONIZED,
 432                                domain, 0);
 433         uint64_t addr;
 434         addr = enc->ws->buffer_get_virtual_address(buf);
 435         addr = addr + offset;
 436         RADEON_ENC_CS(addr >> 32);
 437         RADEON_ENC_CS(addr);
 438 }
 439
 440 void radeon_enc_set_emulation_prevention(struct radeon_encoder *enc, bool set)
 441 {
 442         if (set != enc->emulation_prevention) {
 443                 enc->emulation_prevention = set;
 444                 enc->num_zeros = 0;
 445         }
 446 }
 447
 448 void radeon_enc_output_one_byte(struct radeon_encoder *enc, unsigned char byte)
 449 {
 450         if (enc->byte_index == 0)
 451                 enc->cs->current.buf[enc->cs->current.cdw] = 0;
 452         enc->cs->current.buf[enc->cs->current.cdw] |= ((unsigned int)(byte) << index_to_shifts[enc->byte_index]);
 453         enc->byte_index++;
 454
 455         if (enc->byte_index >= 4) {
 456                 enc->byte_index = 0;
 457                 enc->cs->current.cdw++;
 458         }
 459 }
 460
 461 void radeon_enc_emulation_prevention(struct radeon_encoder *enc, unsigned char byte)
 462 {
 463         if(enc->emulation_prevention) {
 464                 if((enc->num_zeros >= 2) && ((byte == 0x00) || (byte == 0x01) || (byte == 0x03))) {
 465             radeon_enc_output_one_byte(enc, 0x03);
 466             enc->bits_output += 8;
 467             enc->num_zeros = 0;
 468         }
 469         enc->num_zeros = (byte == 0 ? (enc->num_zeros + 1) : 0);
 470     }
 471 }
 472
 473 void radeon_enc_code_fixed_bits(struct radeon_encoder *enc, unsigned int value, unsigned int num_bits)
 474 {
 475         unsigned int bits_to_pack = 0;
 476
 477         while(num_bits > 0) {
 478                 unsigned int value_to_pack = value & (0xffffffff >> (32 - num_bits));
 479                 bits_to_pack = num_bits > (32 - enc->bits_in_shifter) ? (32 - enc->bits_in_shifter) : num_bits;
 480
 481                 if (bits_to_pack < num_bits)
 482                         value_to_pack = value_to_pack >> (num_bits - bits_to_pack);
 483
 484                 enc->shifter |= value_to_pack << (32 - enc->bits_in_shifter - bits_to_pack);
 485                 num_bits -= bits_to_pack;
 486                 enc->bits_in_shifter += bits_to_pack;
 487
 488                 while(enc->bits_in_shifter >= 8) {
 489                         unsigned char output_byte = (unsigned char)(enc->shifter >> 24);
 490                         enc->shifter <<= 8;
 491                         radeon_enc_emulation_prevention(enc, output_byte);
 492                         radeon_enc_output_one_byte(enc, output_byte);
 493                         enc->bits_in_shifter -= 8;
 494                         enc->bits_output += 8;
 495                 }
 496         }
 497 }
 498
 499 void radeon_enc_reset(struct radeon_encoder *enc)
 500 {
 501         enc->emulation_prevention = false;
 502         enc->shifter = 0;
 503         enc->bits_in_shifter = 0;
 504         enc->bits_output = 0;
 505         enc->num_zeros = 0;
 506         enc->byte_index = 0;
 507 }
 508
 509 void radeon_enc_byte_align(struct radeon_encoder *enc)
 510 {
 511         unsigned int num_padding_zeros = (32 - enc->bits_in_shifter) % 8;
 512
 513         if (num_padding_zeros > 0)
 514                 radeon_enc_code_fixed_bits(enc, 0, num_padding_zeros);
 515 }
 516
 517 void radeon_enc_flush_headers(struct radeon_encoder *enc)
 518 {
 519         if (enc->bits_in_shifter != 0) {
 520                 unsigned char output_byte = (unsigned char)(enc->shifter >> 24);
 521                 radeon_enc_emulation_prevention(enc, output_byte);
 522                 radeon_enc_output_one_byte(enc, output_byte);
 523                 enc->bits_output += enc->bits_in_shifter;
 524                 enc->shifter = 0;
 525                 enc->bits_in_shifter = 0;
 526                 enc->num_zeros = 0;
 527         }
 528
 529         if (enc->byte_index > 0) {
 530                 enc->cs->current.cdw++;
 531                 enc->byte_index = 0;
 532         }
 533 }
 534
 535 void radeon_enc_code_ue(struct radeon_encoder *enc, unsigned int value)
 536 {
 537         int x = -1;
 538         unsigned int ue_code = value + 1;
 539         value += 1;
 540
 541         while (value) {
 542                 value = (value >> 1);
 543                 x += 1;
 544         }
 545
 546         unsigned int ue_length = (x << 1) + 1;
 547         radeon_enc_code_fixed_bits(enc, ue_code, ue_length);
 548 }
 549
 550 void radeon_enc_code_se(struct radeon_encoder *enc, int value)
 551 {
 552         unsigned int v = 0;
 553
 554         if (value != 0)
 555                 v = (value < 0 ? ((unsigned int)(0 - value) << 1) : (((unsigned int)(value) << 1) - 1));
 556
 557         radeon_enc_code_ue(enc, v);
 558 }