src/gallium/drivers/radeon/radeon_vcn_enc.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2017 Advanced Micro Devices, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include <stdio.h>
  29
  30 #include "pipe/p_video_codec.h"
  31
  32 #include "util/u_video.h"
  33 #include "util/u_memory.h"
  34
  35 #include "vl/vl_video_buffer.h"
  36
  37 #include "radeonsi/si_pipe.h"
  38 #include "radeon_video.h"
  39 #include "radeon_vcn_enc.h"
  40
  41 static const unsigned index_to_shifts[4] = {24, 16, 8, 0};
  42
  43 static void radeon_vcn_enc_get_param(struct radeon_encoder *enc, struct pipe_picture_desc *picture)
  44 {
  45    if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_MPEG4_AVC) {
  46       struct pipe_h264_enc_picture_desc *pic = (struct pipe_h264_enc_picture_desc *)picture;
  47       enc->enc_pic.picture_type = pic->picture_type;
  48       enc->enc_pic.frame_num = pic->frame_num;
  49       enc->enc_pic.pic_order_cnt = pic->pic_order_cnt;
  50       enc->enc_pic.pic_order_cnt_type = pic->pic_order_cnt_type;
  51       enc->enc_pic.ref_idx_l0 = pic->ref_idx_l0;
  52       enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1;
  53       enc->enc_pic.not_referenced = pic->not_referenced;
  54       enc->enc_pic.is_idr = (pic->picture_type == PIPE_H264_ENC_PICTURE_TYPE_IDR);
  55       enc->enc_pic.crop_left = 0;
  56       enc->enc_pic.crop_right = (align(enc->base.width, 16) - enc->base.width) / 2;
  57       enc->enc_pic.crop_top = 0;
  58       enc->enc_pic.crop_bottom = (align(enc->base.height, 16) - enc->base.height) / 2;
  59       enc->enc_pic.rc_layer_init.target_bit_rate = pic->rate_ctrl.target_bitrate;
  60       enc->enc_pic.rc_layer_init.peak_bit_rate = pic->rate_ctrl.peak_bitrate;
  61       enc->enc_pic.rc_layer_init.frame_rate_num = pic->rate_ctrl.frame_rate_num;
  62       enc->enc_pic.rc_layer_init.frame_rate_den = pic->rate_ctrl.frame_rate_den;
  63       enc->enc_pic.rc_layer_init.vbv_buffer_size = pic->rate_ctrl.vbv_buffer_size;
  64       enc->enc_pic.rc_layer_init.avg_target_bits_per_picture = pic->rate_ctrl.target_bits_picture;
  65       enc->enc_pic.rc_layer_init.peak_bits_per_picture_integer = pic->rate_ctrl.peak_bits_picture_integer;
  66       enc->enc_pic.rc_layer_init.peak_bits_per_picture_fractional = pic->rate_ctrl.peak_bits_picture_fraction;
  67       enc->enc_pic.rc_session_init.vbv_buffer_level = pic->rate_ctrl.vbv_buf_lv;
  68       enc->enc_pic.rc_per_pic.qp = pic->quant_i_frames;
  69       enc->enc_pic.rc_per_pic.min_qp_app = 0;
  70       enc->enc_pic.rc_per_pic.max_qp_app = 51;
  71       enc->enc_pic.rc_per_pic.max_au_size = 0;
  72       enc->enc_pic.rc_per_pic.enabled_filler_data = pic->rate_ctrl.fill_data_enable;
  73       enc->enc_pic.rc_per_pic.skip_frame_enable = false;
  74       enc->enc_pic.rc_per_pic.enforce_hrd = pic->rate_ctrl.enforce_hrd;
  75       switch(pic->rate_ctrl.rate_ctrl_method) {
  76          case PIPE_H264_ENC_RATE_CONTROL_METHOD_DISABLE:
  77             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
  78             break;
  79          case PIPE_H264_ENC_RATE_CONTROL_METHOD_CONSTANT_SKIP:
  80          case PIPE_H264_ENC_RATE_CONTROL_METHOD_CONSTANT:
  81             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_CBR;
  82             break;
  83          case PIPE_H264_ENC_RATE_CONTROL_METHOD_VARIABLE_SKIP:
  84          case PIPE_H264_ENC_RATE_CONTROL_METHOD_VARIABLE:
  85             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_PEAK_CONSTRAINED_VBR;
  86             break;
  87          default:
  88             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
  89       }
  90    } else if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_HEVC) {
  91       struct pipe_h265_enc_picture_desc *pic = (struct pipe_h265_enc_picture_desc *)picture;
  92       enc->enc_pic.picture_type = pic->picture_type;
  93       enc->enc_pic.frame_num = pic->frame_num;
  94       enc->enc_pic.pic_order_cnt = pic->pic_order_cnt;
  95       enc->enc_pic.pic_order_cnt_type = pic->pic_order_cnt_type;
  96       enc->enc_pic.ref_idx_l0 = pic->ref_idx_l0;
  97       enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1;
  98       enc->enc_pic.not_referenced = pic->not_referenced;
  99       enc->enc_pic.is_idr = (pic->picture_type == PIPE_H265_ENC_PICTURE_TYPE_IDR) ||
 100                             (pic->picture_type == PIPE_H265_ENC_PICTURE_TYPE_I);
 101       enc->enc_pic.crop_left = 0;
 102       enc->enc_pic.crop_right = (align(enc->base.width, 16) - enc->base.width) / 2;
 103       enc->enc_pic.crop_top = 0;
 104       enc->enc_pic.crop_bottom = (align(enc->base.height, 16) - enc->base.height) / 2;
 105       enc->enc_pic.general_tier_flag = pic->seq.general_tier_flag;
 106       enc->enc_pic.general_profile_idc = pic->seq.general_profile_idc;
 107       enc->enc_pic.general_level_idc = pic->seq.general_level_idc;
 108       enc->enc_pic.max_poc =
 109          MAX2(16, util_next_power_of_two(pic->seq.intra_period));
 110       enc->enc_pic.log2_max_poc = 0;
 111       for (int i = enc->enc_pic.max_poc; i != 0; enc->enc_pic.log2_max_poc++)
 112          i = (i >> 1);
 113       enc->enc_pic.chroma_format_idc = pic->seq.chroma_format_idc;
 114       enc->enc_pic.pic_width_in_luma_samples = pic->seq.pic_width_in_luma_samples;
 115       enc->enc_pic.pic_height_in_luma_samples = pic->seq.pic_height_in_luma_samples;
 116       enc->enc_pic.log2_diff_max_min_luma_coding_block_size = pic->seq.log2_diff_max_min_luma_coding_block_size;
 117       enc->enc_pic.log2_min_transform_block_size_minus2 = pic->seq.log2_min_transform_block_size_minus2;
 118       enc->enc_pic.log2_diff_max_min_transform_block_size = pic->seq.log2_diff_max_min_transform_block_size;
 119       enc->enc_pic.max_transform_hierarchy_depth_inter = pic->seq.max_transform_hierarchy_depth_inter;
 120       enc->enc_pic.max_transform_hierarchy_depth_intra = pic->seq.max_transform_hierarchy_depth_intra;
 121       enc->enc_pic.log2_parallel_merge_level_minus2 = pic->pic.log2_parallel_merge_level_minus2;
 122       enc->enc_pic.bit_depth_luma_minus8 = pic->seq.bit_depth_luma_minus8;
 123       enc->enc_pic.bit_depth_chroma_minus8 = pic->seq.bit_depth_chroma_minus8;
 124       enc->enc_pic.nal_unit_type = pic->pic.nal_unit_type;
 125       enc->enc_pic.max_num_merge_cand = pic->slice.max_num_merge_cand;
 126       enc->enc_pic.sample_adaptive_offset_enabled_flag = pic->seq.sample_adaptive_offset_enabled_flag;
 127       enc->enc_pic.pcm_enabled_flag = pic->seq.pcm_enabled_flag;
 128       enc->enc_pic.sps_temporal_mvp_enabled_flag = pic->seq.sps_temporal_mvp_enabled_flag;
 129       enc->enc_pic.hevc_deblock.loop_filter_across_slices_enabled = pic->slice.slice_loop_filter_across_slices_enabled_flag;
 130       enc->enc_pic.hevc_deblock.deblocking_filter_disabled = pic->slice.slice_deblocking_filter_disabled_flag;
 131       enc->enc_pic.hevc_deblock.beta_offset_div2 = pic->slice.slice_beta_offset_div2;
 132       enc->enc_pic.hevc_deblock.tc_offset_div2 = pic->slice.slice_tc_offset_div2;
 133       enc->enc_pic.hevc_deblock.cb_qp_offset = pic->slice.slice_cb_qp_offset;
 134       enc->enc_pic.hevc_deblock.cr_qp_offset = pic->slice.slice_cr_qp_offset;
 135       enc->enc_pic.hevc_spec_misc.log2_min_luma_coding_block_size_minus3 = pic->seq.log2_min_luma_coding_block_size_minus3;
 136       enc->enc_pic.hevc_spec_misc.amp_disabled = !pic->seq.amp_enabled_flag;
 137       enc->enc_pic.hevc_spec_misc.strong_intra_smoothing_enabled = pic->seq.strong_intra_smoothing_enabled_flag;
 138       enc->enc_pic.hevc_spec_misc.constrained_intra_pred_flag = pic->pic.constrained_intra_pred_flag;
 139       enc->enc_pic.hevc_spec_misc.cabac_init_flag = pic->slice.cabac_init_flag;
 140       enc->enc_pic.hevc_spec_misc.half_pel_enabled = 1;
 141       enc->enc_pic.hevc_spec_misc.quarter_pel_enabled = 1;
 142       enc->enc_pic.rc_layer_init.target_bit_rate = pic->rc.target_bitrate;
 143       enc->enc_pic.rc_layer_init.peak_bit_rate = pic->rc.peak_bitrate;
 144       enc->enc_pic.rc_layer_init.frame_rate_num = pic->rc.frame_rate_num;
 145       enc->enc_pic.rc_layer_init.frame_rate_den = pic->rc.frame_rate_den;
 146       enc->enc_pic.rc_layer_init.vbv_buffer_size = pic->rc.vbv_buffer_size;
 147       enc->enc_pic.rc_layer_init.avg_target_bits_per_picture = pic->rc.target_bits_picture;
 148       enc->enc_pic.rc_layer_init.peak_bits_per_picture_integer = pic->rc.peak_bits_picture_integer;
 149       enc->enc_pic.rc_layer_init.peak_bits_per_picture_fractional = pic->rc.peak_bits_picture_fraction;
 150       enc->enc_pic.rc_session_init.vbv_buffer_level = pic->rc.vbv_buf_lv;
 151       enc->enc_pic.rc_per_pic.qp = pic->rc.quant_i_frames;
 152       enc->enc_pic.rc_per_pic.min_qp_app = 0;
 153       enc->enc_pic.rc_per_pic.max_qp_app = 51;
 154       enc->enc_pic.rc_per_pic.max_au_size = 0;
 155       enc->enc_pic.rc_per_pic.enabled_filler_data = pic->rc.fill_data_enable;
 156       enc->enc_pic.rc_per_pic.skip_frame_enable = false;
 157       enc->enc_pic.rc_per_pic.enforce_hrd = pic->rc.enforce_hrd;
 158       switch(pic->rc.rate_ctrl_method) {
 159          case PIPE_H265_ENC_RATE_CONTROL_METHOD_DISABLE:
 160          enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
 161             break;
 162          case PIPE_H265_ENC_RATE_CONTROL_METHOD_CONSTANT_SKIP:
 163          case PIPE_H265_ENC_RATE_CONTROL_METHOD_CONSTANT:
 164             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_CBR;
 165             break;
 166          case PIPE_H265_ENC_RATE_CONTROL_METHOD_VARIABLE_SKIP:
 167          case PIPE_H265_ENC_RATE_CONTROL_METHOD_VARIABLE:
 168             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_PEAK_CONSTRAINED_VBR;
 169             break;
 170          default:
 171             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
 172       }
 173    }
 174 }
 175
 176 static void flush(struct radeon_encoder *enc)
 177 {
 178         enc->ws->cs_flush(enc->cs, PIPE_FLUSH_ASYNC, NULL);
 179 }
 180
 181 static void radeon_enc_flush(struct pipe_video_codec *encoder)
 182 {
 183         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 184         flush(enc);
 185 }
 186
 187 static void radeon_enc_cs_flush(void *ctx, unsigned flags,
 188                                                                 struct pipe_fence_handle **fence)
 189 {
 190         // just ignored
 191 }
 192
 193 static unsigned get_cpb_num(struct radeon_encoder *enc)
 194 {
 195         unsigned w = align(enc->base.width, 16) / 16;
 196         unsigned h = align(enc->base.height, 16) / 16;
 197         unsigned dpb;
 198
 199         switch (enc->base.level) {
 200         case 10:
 201                 dpb = 396;
 202                 break;
 203         case 11:
 204                 dpb = 900;
 205                 break;
 206         case 12:
 207         case 13:
 208         case 20:
 209                 dpb = 2376;
 210                 break;
 211         case 21:
 212                 dpb = 4752;
 213                 break;
 214         case 22:
 215         case 30:
 216                 dpb = 8100;
 217                 break;
 218         case 31:
 219                 dpb = 18000;
 220                 break;
 221         case 32:
 222                 dpb = 20480;
 223                 break;
 224         case 40:
 225         case 41:
 226                 dpb = 32768;
 227                 break;
 228         case 42:
 229                 dpb = 34816;
 230                 break;
 231         case 50:
 232                 dpb = 110400;
 233                 break;
 234         default:
 235         case 51:
 236         case 52:
 237                 dpb = 184320;
 238                 break;
 239         }
 240
 241         return MIN2(dpb / (w * h), 16);
 242 }
 243
 244 static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
 245                                                          struct pipe_video_buffer *source,
 246                                                          struct pipe_picture_desc *picture)
 247 {
 248         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 249         struct vl_video_buffer *vid_buf = (struct vl_video_buffer *)source;
 250
 251         radeon_vcn_enc_get_param(enc, picture);
 252
 253         enc->get_buffer(vid_buf->resources[0], &enc->handle, &enc->luma);
 254         enc->get_buffer(vid_buf->resources[1], NULL, &enc->chroma);
 255
 256         enc->need_feedback = false;
 257
 258         if (!enc->stream_handle) {
 259                 struct rvid_buffer fb;
 260                 enc->stream_handle = si_vid_alloc_stream_handle();
 261                 enc->si = CALLOC_STRUCT(rvid_buffer);
 262                 si_vid_create_buffer(enc->screen, enc->si, 128 * 1024, PIPE_USAGE_STAGING);
 263                 si_vid_create_buffer(enc->screen, &fb, 4096, PIPE_USAGE_STAGING);
 264                 enc->fb = &fb;
 265                 enc->begin(enc);
 266                 flush(enc);
 267                 si_vid_destroy_buffer(&fb);
 268         }
 269 }
 270
 271 static void radeon_enc_encode_bitstream(struct pipe_video_codec *encoder,
 272                                                                   struct pipe_video_buffer *source,
 273                                                                   struct pipe_resource *destination,
 274                                                                   void **fb)
 275 {
 276         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 277         enc->get_buffer(destination, &enc->bs_handle, NULL);
 278         enc->bs_size = destination->width0;
 279
 280         *fb = enc->fb = CALLOC_STRUCT(rvid_buffer);
 281
 282         if (!si_vid_create_buffer(enc->screen, enc->fb, 4096, PIPE_USAGE_STAGING)) {
 283                 RVID_ERR("Can't create feedback buffer.\n");
 284                 return;
 285         }
 286
 287         enc->need_feedback = true;
 288         enc->encode(enc);
 289 }
 290
 291 static void radeon_enc_end_frame(struct pipe_video_codec *encoder,
 292                                                    struct pipe_video_buffer *source,
 293                                                    struct pipe_picture_desc *picture)
 294 {
 295         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 296         flush(enc);
 297 }
 298
 299 static void radeon_enc_destroy(struct pipe_video_codec *encoder)
 300 {
 301         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 302
 303         if (enc->stream_handle) {
 304                 struct rvid_buffer fb;
 305                 enc->need_feedback = false;
 306                 si_vid_create_buffer(enc->screen, &fb, 512, PIPE_USAGE_STAGING);
 307                 enc->fb = &fb;
 308                 enc->destroy(enc);
 309                 flush(enc);
 310                 si_vid_destroy_buffer(&fb);
 311         }
 312
 313         si_vid_destroy_buffer(&enc->cpb);
 314         enc->ws->cs_destroy(enc->cs);
 315         FREE(enc);
 316 }
 317
 318 static void radeon_enc_get_feedback(struct pipe_video_codec *encoder,
 319                                                           void *feedback, unsigned *size)
 320 {
 321         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 322         struct rvid_buffer *fb = feedback;
 323
 324         if (size) {
 325                 uint32_t *ptr = enc->ws->buffer_map(
 326                         fb->res->buf, enc->cs,
 327                         PIPE_TRANSFER_READ_WRITE | RADEON_TRANSFER_TEMPORARY);
 328                 if (ptr[1])
 329                         *size = ptr[6];
 330                 else
 331                         *size = 0;
 332                 enc->ws->buffer_unmap(fb->res->buf);
 333         }
 334
 335         si_vid_destroy_buffer(fb);
 336         FREE(fb);
 337 }
 338
 339 struct pipe_video_codec *radeon_create_encoder(struct pipe_context *context,
 340                 const struct pipe_video_codec *templ,
 341                 struct radeon_winsys* ws,
 342                 radeon_enc_get_buffer get_buffer)
 343 {
 344         struct si_screen *sscreen = (struct si_screen *)context->screen;
 345         struct si_context *sctx = (struct si_context*)context;
 346         struct radeon_encoder *enc;
 347         struct pipe_video_buffer *tmp_buf, templat = {};
 348         struct radeon_surf *tmp_surf;
 349         unsigned cpb_size;
 350
 351         enc = CALLOC_STRUCT(radeon_encoder);
 352
 353         if (!enc)
 354                 return NULL;
 355
 356         enc->alignment = 256;
 357         enc->base = *templ;
 358         enc->base.context = context;
 359         enc->base.destroy = radeon_enc_destroy;
 360         enc->base.begin_frame = radeon_enc_begin_frame;
 361         enc->base.encode_bitstream = radeon_enc_encode_bitstream;
 362         enc->base.end_frame = radeon_enc_end_frame;
 363         enc->base.flush = radeon_enc_flush;
 364         enc->base.get_feedback = radeon_enc_get_feedback;
 365         enc->get_buffer = get_buffer;
 366         enc->bits_in_shifter = 0;
 367         enc->screen = context->screen;
 368         enc->ws = ws;
 369         enc->cs = ws->cs_create(sctx->ctx, RING_VCN_ENC, radeon_enc_cs_flush,
 370                                 enc, false);
 371
 372         if (!enc->cs) {
 373                 RVID_ERR("Can't get command submission context.\n");
 374                 goto error;
 375         }
 376
 377         struct rvid_buffer si;
 378         si_vid_create_buffer(enc->screen, &si, 128 * 1024, PIPE_USAGE_STAGING);
 379         enc->si = &si;
 380
 381         templat.buffer_format = PIPE_FORMAT_NV12;
 382         templat.chroma_format = PIPE_VIDEO_CHROMA_FORMAT_420;
 383         templat.width = enc->base.width;
 384         templat.height = enc->base.height;
 385         templat.interlaced = false;
 386
 387         if (!(tmp_buf = context->create_video_buffer(context, &templat))) {
 388                 RVID_ERR("Can't create video buffer.\n");
 389                 goto error;
 390         }
 391
 392         enc->cpb_num = get_cpb_num(enc);
 393
 394         if (!enc->cpb_num)
 395                 goto error;
 396
 397         get_buffer(((struct vl_video_buffer *)tmp_buf)->resources[0], NULL, &tmp_surf);
 398
 399         cpb_size = (sscreen->info.chip_class < GFX9) ?
 400                            align(tmp_surf->u.legacy.level[0].nblk_x * tmp_surf->bpe, 128) *
 401                            align(tmp_surf->u.legacy.level[0].nblk_y, 32) :
 402                            align(tmp_surf->u.gfx9.surf_pitch * tmp_surf->bpe, 256) *
 403                            align(tmp_surf->u.gfx9.surf_height, 32);
 404
 405         cpb_size = cpb_size * 3 / 2;
 406         cpb_size = cpb_size * enc->cpb_num;
 407         tmp_buf->destroy(tmp_buf);
 408
 409         if (!si_vid_create_buffer(enc->screen, &enc->cpb, cpb_size, PIPE_USAGE_DEFAULT)) {
 410                 RVID_ERR("Can't create CPB buffer.\n");
 411                 goto error;
 412         }
 413
 414         if (sscreen->info.family <= CHIP_RAVEN2)
 415                 radeon_enc_1_2_init(enc);
 416         else
 417                 radeon_enc_2_0_init(enc);
 418
 419         return &enc->base;
 420
 421 error:
 422         if (enc->cs)
 423                 enc->ws->cs_destroy(enc->cs);
 424
 425         si_vid_destroy_buffer(&enc->cpb);
 426
 427         FREE(enc);
 428         return NULL;
 429 }
 430
 431 void radeon_enc_add_buffer(struct radeon_encoder *enc, struct pb_buffer *buf,
 432                            enum radeon_bo_usage usage, enum radeon_bo_domain domain,
 433                            signed offset)
 434 {
 435         enc->ws->cs_add_buffer(enc->cs, buf, usage | RADEON_USAGE_SYNCHRONIZED,
 436                                domain, 0);
 437         uint64_t addr;
 438         addr = enc->ws->buffer_get_virtual_address(buf);
 439         addr = addr + offset;
 440         RADEON_ENC_CS(addr >> 32);
 441         RADEON_ENC_CS(addr);
 442 }
 443
 444 void radeon_enc_set_emulation_prevention(struct radeon_encoder *enc, bool set)
 445 {
 446         if (set != enc->emulation_prevention) {
 447                 enc->emulation_prevention = set;
 448                 enc->num_zeros = 0;
 449         }
 450 }
 451
 452 void radeon_enc_output_one_byte(struct radeon_encoder *enc, unsigned char byte)
 453 {
 454         if (enc->byte_index == 0)
 455                 enc->cs->current.buf[enc->cs->current.cdw] = 0;
 456         enc->cs->current.buf[enc->cs->current.cdw] |= ((unsigned int)(byte) << index_to_shifts[enc->byte_index]);
 457         enc->byte_index++;
 458
 459         if (enc->byte_index >= 4) {
 460                 enc->byte_index = 0;
 461                 enc->cs->current.cdw++;
 462         }
 463 }
 464
 465 void radeon_enc_emulation_prevention(struct radeon_encoder *enc, unsigned char byte)
 466 {
 467         if(enc->emulation_prevention) {
 468                 if((enc->num_zeros >= 2) && ((byte == 0x00) || (byte == 0x01) || (byte == 0x03))) {
 469             radeon_enc_output_one_byte(enc, 0x03);
 470             enc->bits_output += 8;
 471             enc->num_zeros = 0;
 472         }
 473         enc->num_zeros = (byte == 0 ? (enc->num_zeros + 1) : 0);
 474     }
 475 }
 476
 477 void radeon_enc_code_fixed_bits(struct radeon_encoder *enc, unsigned int value, unsigned int num_bits)
 478 {
 479         unsigned int bits_to_pack = 0;
 480
 481         while(num_bits > 0) {
 482                 unsigned int value_to_pack = value & (0xffffffff >> (32 - num_bits));
 483                 bits_to_pack = num_bits > (32 - enc->bits_in_shifter) ? (32 - enc->bits_in_shifter) : num_bits;
 484
 485                 if (bits_to_pack < num_bits)
 486                         value_to_pack = value_to_pack >> (num_bits - bits_to_pack);
 487
 488                 enc->shifter |= value_to_pack << (32 - enc->bits_in_shifter - bits_to_pack);
 489                 num_bits -= bits_to_pack;
 490                 enc->bits_in_shifter += bits_to_pack;
 491
 492                 while(enc->bits_in_shifter >= 8) {
 493                         unsigned char output_byte = (unsigned char)(enc->shifter >> 24);
 494                         enc->shifter <<= 8;
 495                         radeon_enc_emulation_prevention(enc, output_byte);
 496                         radeon_enc_output_one_byte(enc, output_byte);
 497                         enc->bits_in_shifter -= 8;
 498                         enc->bits_output += 8;
 499                 }
 500         }
 501 }
 502
 503 void radeon_enc_reset(struct radeon_encoder *enc)
 504 {
 505         enc->emulation_prevention = false;
 506         enc->shifter = 0;
 507         enc->bits_in_shifter = 0;
 508         enc->bits_output = 0;
 509         enc->num_zeros = 0;
 510         enc->byte_index = 0;
 511 }
 512
 513 void radeon_enc_byte_align(struct radeon_encoder *enc)
 514 {
 515         unsigned int num_padding_zeros = (32 - enc->bits_in_shifter) % 8;
 516
 517         if (num_padding_zeros > 0)
 518                 radeon_enc_code_fixed_bits(enc, 0, num_padding_zeros);
 519 }
 520
 521 void radeon_enc_flush_headers(struct radeon_encoder *enc)
 522 {
 523         if (enc->bits_in_shifter != 0) {
 524                 unsigned char output_byte = (unsigned char)(enc->shifter >> 24);
 525                 radeon_enc_emulation_prevention(enc, output_byte);
 526                 radeon_enc_output_one_byte(enc, output_byte);
 527                 enc->bits_output += enc->bits_in_shifter;
 528                 enc->shifter = 0;
 529                 enc->bits_in_shifter = 0;
 530                 enc->num_zeros = 0;
 531         }
 532
 533         if (enc->byte_index > 0) {
 534                 enc->cs->current.cdw++;
 535                 enc->byte_index = 0;
 536         }
 537 }
 538
 539 void radeon_enc_code_ue(struct radeon_encoder *enc, unsigned int value)
 540 {
 541         int x = -1;
 542         unsigned int ue_code = value + 1;
 543         value += 1;
 544
 545         while (value) {
 546                 value = (value >> 1);
 547                 x += 1;
 548         }
 549
 550         unsigned int ue_length = (x << 1) + 1;
 551         radeon_enc_code_fixed_bits(enc, ue_code, ue_length);
 552 }
 553
 554 void radeon_enc_code_se(struct radeon_encoder *enc, int value)
 555 {
 556         unsigned int v = 0;
 557
 558         if (value != 0)
 559                 v = (value < 0 ? ((unsigned int)(0 - value) << 1) : (((unsigned int)(value) << 1) - 1));
 560
 561         radeon_enc_code_ue(enc, v);
 562 }