src/gallium/drivers/radeon/radeon_vcn_enc.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2017 Advanced Micro Devices, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include <stdio.h>
  29
  30 #include "pipe/p_video_codec.h"
  31
  32 #include "util/u_video.h"
  33 #include "util/u_memory.h"
  34
  35 #include "vl/vl_video_buffer.h"
  36
  37 #include "radeonsi/si_pipe.h"
  38 #include "radeon_video.h"
  39 #include "radeon_vcn_enc.h"
  40
  41 static const unsigned index_to_shifts[4] = {24, 16, 8, 0};
  42
  43 static void radeon_vcn_enc_get_param(struct radeon_encoder *enc, struct pipe_picture_desc *picture)
  44 {
  45    if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_MPEG4_AVC) {
  46       struct pipe_h264_enc_picture_desc *pic = (struct pipe_h264_enc_picture_desc *)picture;
  47       enc->enc_pic.picture_type = pic->picture_type;
  48       enc->enc_pic.frame_num = pic->frame_num;
  49       enc->enc_pic.pic_order_cnt = pic->pic_order_cnt;
  50       enc->enc_pic.pic_order_cnt_type = pic->pic_order_cnt_type;
  51       enc->enc_pic.ref_idx_l0 = pic->ref_idx_l0;
  52       enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1;
  53       enc->enc_pic.not_referenced = pic->not_referenced;
  54       enc->enc_pic.is_idr = (pic->picture_type == PIPE_H264_ENC_PICTURE_TYPE_IDR);
  55       if (pic->pic_ctrl.enc_frame_cropping_flag) {
  56           enc->enc_pic.crop_left = pic->pic_ctrl.enc_frame_crop_left_offset;
  57           enc->enc_pic.crop_right = pic->pic_ctrl.enc_frame_crop_right_offset;
  58           enc->enc_pic.crop_top = pic->pic_ctrl.enc_frame_crop_top_offset;
  59           enc->enc_pic.crop_bottom = pic->pic_ctrl.enc_frame_crop_bottom_offset;
  60       } else {
  61           enc->enc_pic.crop_left = 0;
  62           enc->enc_pic.crop_right = (align(enc->base.width, 16) - enc->base.width) / 2;
  63           enc->enc_pic.crop_top = 0;
  64           enc->enc_pic.crop_bottom = (align(enc->base.height, 16) - enc->base.height) / 2;
  65       }
  66       enc->enc_pic.rc_layer_init.target_bit_rate = pic->rate_ctrl.target_bitrate;
  67       enc->enc_pic.rc_layer_init.peak_bit_rate = pic->rate_ctrl.peak_bitrate;
  68       enc->enc_pic.rc_layer_init.frame_rate_num = pic->rate_ctrl.frame_rate_num;
  69       enc->enc_pic.rc_layer_init.frame_rate_den = pic->rate_ctrl.frame_rate_den;
  70       enc->enc_pic.rc_layer_init.vbv_buffer_size = pic->rate_ctrl.vbv_buffer_size;
  71       enc->enc_pic.rc_layer_init.avg_target_bits_per_picture = pic->rate_ctrl.target_bits_picture;
  72       enc->enc_pic.rc_layer_init.peak_bits_per_picture_integer = pic->rate_ctrl.peak_bits_picture_integer;
  73       enc->enc_pic.rc_layer_init.peak_bits_per_picture_fractional = pic->rate_ctrl.peak_bits_picture_fraction;
  74       enc->enc_pic.rc_session_init.vbv_buffer_level = pic->rate_ctrl.vbv_buf_lv;
  75       enc->enc_pic.rc_per_pic.qp = pic->quant_i_frames;
  76       enc->enc_pic.rc_per_pic.min_qp_app = 0;
  77       enc->enc_pic.rc_per_pic.max_qp_app = 51;
  78       enc->enc_pic.rc_per_pic.max_au_size = 0;
  79       enc->enc_pic.rc_per_pic.enabled_filler_data = pic->rate_ctrl.fill_data_enable;
  80       enc->enc_pic.rc_per_pic.skip_frame_enable = false;
  81       enc->enc_pic.rc_per_pic.enforce_hrd = pic->rate_ctrl.enforce_hrd;
  82       switch(pic->rate_ctrl.rate_ctrl_method) {
  83          case PIPE_H264_ENC_RATE_CONTROL_METHOD_DISABLE:
  84             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
  85             break;
  86          case PIPE_H264_ENC_RATE_CONTROL_METHOD_CONSTANT_SKIP:
  87          case PIPE_H264_ENC_RATE_CONTROL_METHOD_CONSTANT:
  88             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_CBR;
  89             break;
  90          case PIPE_H264_ENC_RATE_CONTROL_METHOD_VARIABLE_SKIP:
  91          case PIPE_H264_ENC_RATE_CONTROL_METHOD_VARIABLE:
  92             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_PEAK_CONSTRAINED_VBR;
  93             break;
  94          default:
  95             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
  96       }
  97    } else if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_HEVC) {
  98       struct pipe_h265_enc_picture_desc *pic = (struct pipe_h265_enc_picture_desc *)picture;
  99       enc->enc_pic.picture_type = pic->picture_type;
 100       enc->enc_pic.frame_num = pic->frame_num;
 101       enc->enc_pic.pic_order_cnt = pic->pic_order_cnt;
 102       enc->enc_pic.pic_order_cnt_type = pic->pic_order_cnt_type;
 103       enc->enc_pic.ref_idx_l0 = pic->ref_idx_l0;
 104       enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1;
 105       enc->enc_pic.not_referenced = pic->not_referenced;
 106       enc->enc_pic.is_idr = (pic->picture_type == PIPE_H265_ENC_PICTURE_TYPE_IDR) ||
 107                             (pic->picture_type == PIPE_H265_ENC_PICTURE_TYPE_I);
 108       enc->enc_pic.crop_left = 0;
 109       enc->enc_pic.crop_right = (align(enc->base.width, 16) - enc->base.width) / 2;
 110       enc->enc_pic.crop_top = 0;
 111       enc->enc_pic.crop_bottom = (align(enc->base.height, 16) - enc->base.height) / 2;
 112       enc->enc_pic.general_tier_flag = pic->seq.general_tier_flag;
 113       enc->enc_pic.general_profile_idc = pic->seq.general_profile_idc;
 114       enc->enc_pic.general_level_idc = pic->seq.general_level_idc;
 115       enc->enc_pic.max_poc =
 116          MAX2(16, util_next_power_of_two(pic->seq.intra_period));
 117       enc->enc_pic.log2_max_poc = 0;
 118       for (int i = enc->enc_pic.max_poc; i != 0; enc->enc_pic.log2_max_poc++)
 119          i = (i >> 1);
 120       enc->enc_pic.chroma_format_idc = pic->seq.chroma_format_idc;
 121       enc->enc_pic.pic_width_in_luma_samples = pic->seq.pic_width_in_luma_samples;
 122       enc->enc_pic.pic_height_in_luma_samples = pic->seq.pic_height_in_luma_samples;
 123       enc->enc_pic.log2_diff_max_min_luma_coding_block_size = pic->seq.log2_diff_max_min_luma_coding_block_size;
 124       enc->enc_pic.log2_min_transform_block_size_minus2 = pic->seq.log2_min_transform_block_size_minus2;
 125       enc->enc_pic.log2_diff_max_min_transform_block_size = pic->seq.log2_diff_max_min_transform_block_size;
 126       enc->enc_pic.max_transform_hierarchy_depth_inter = pic->seq.max_transform_hierarchy_depth_inter;
 127       enc->enc_pic.max_transform_hierarchy_depth_intra = pic->seq.max_transform_hierarchy_depth_intra;
 128       enc->enc_pic.log2_parallel_merge_level_minus2 = pic->pic.log2_parallel_merge_level_minus2;
 129       enc->enc_pic.bit_depth_luma_minus8 = pic->seq.bit_depth_luma_minus8;
 130       enc->enc_pic.bit_depth_chroma_minus8 = pic->seq.bit_depth_chroma_minus8;
 131       enc->enc_pic.nal_unit_type = pic->pic.nal_unit_type;
 132       enc->enc_pic.max_num_merge_cand = pic->slice.max_num_merge_cand;
 133       enc->enc_pic.sample_adaptive_offset_enabled_flag = pic->seq.sample_adaptive_offset_enabled_flag;
 134       enc->enc_pic.pcm_enabled_flag = pic->seq.pcm_enabled_flag;
 135       enc->enc_pic.sps_temporal_mvp_enabled_flag = pic->seq.sps_temporal_mvp_enabled_flag;
 136       enc->enc_pic.hevc_deblock.loop_filter_across_slices_enabled = pic->slice.slice_loop_filter_across_slices_enabled_flag;
 137       enc->enc_pic.hevc_deblock.deblocking_filter_disabled = pic->slice.slice_deblocking_filter_disabled_flag;
 138       enc->enc_pic.hevc_deblock.beta_offset_div2 = pic->slice.slice_beta_offset_div2;
 139       enc->enc_pic.hevc_deblock.tc_offset_div2 = pic->slice.slice_tc_offset_div2;
 140       enc->enc_pic.hevc_deblock.cb_qp_offset = pic->slice.slice_cb_qp_offset;
 141       enc->enc_pic.hevc_deblock.cr_qp_offset = pic->slice.slice_cr_qp_offset;
 142       enc->enc_pic.hevc_spec_misc.log2_min_luma_coding_block_size_minus3 = pic->seq.log2_min_luma_coding_block_size_minus3;
 143       enc->enc_pic.hevc_spec_misc.amp_disabled = !pic->seq.amp_enabled_flag;
 144       enc->enc_pic.hevc_spec_misc.strong_intra_smoothing_enabled = pic->seq.strong_intra_smoothing_enabled_flag;
 145       enc->enc_pic.hevc_spec_misc.constrained_intra_pred_flag = pic->pic.constrained_intra_pred_flag;
 146       enc->enc_pic.hevc_spec_misc.cabac_init_flag = pic->slice.cabac_init_flag;
 147       enc->enc_pic.hevc_spec_misc.half_pel_enabled = 1;
 148       enc->enc_pic.hevc_spec_misc.quarter_pel_enabled = 1;
 149       enc->enc_pic.rc_layer_init.target_bit_rate = pic->rc.target_bitrate;
 150       enc->enc_pic.rc_layer_init.peak_bit_rate = pic->rc.peak_bitrate;
 151       enc->enc_pic.rc_layer_init.frame_rate_num = pic->rc.frame_rate_num;
 152       enc->enc_pic.rc_layer_init.frame_rate_den = pic->rc.frame_rate_den;
 153       enc->enc_pic.rc_layer_init.vbv_buffer_size = pic->rc.vbv_buffer_size;
 154       enc->enc_pic.rc_layer_init.avg_target_bits_per_picture = pic->rc.target_bits_picture;
 155       enc->enc_pic.rc_layer_init.peak_bits_per_picture_integer = pic->rc.peak_bits_picture_integer;
 156       enc->enc_pic.rc_layer_init.peak_bits_per_picture_fractional = pic->rc.peak_bits_picture_fraction;
 157       enc->enc_pic.rc_session_init.vbv_buffer_level = pic->rc.vbv_buf_lv;
 158       enc->enc_pic.rc_per_pic.qp = pic->rc.quant_i_frames;
 159       enc->enc_pic.rc_per_pic.min_qp_app = 0;
 160       enc->enc_pic.rc_per_pic.max_qp_app = 51;
 161       enc->enc_pic.rc_per_pic.max_au_size = 0;
 162       enc->enc_pic.rc_per_pic.enabled_filler_data = pic->rc.fill_data_enable;
 163       enc->enc_pic.rc_per_pic.skip_frame_enable = false;
 164       enc->enc_pic.rc_per_pic.enforce_hrd = pic->rc.enforce_hrd;
 165       switch(pic->rc.rate_ctrl_method) {
 166          case PIPE_H265_ENC_RATE_CONTROL_METHOD_DISABLE:
 167          enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
 168             break;
 169          case PIPE_H265_ENC_RATE_CONTROL_METHOD_CONSTANT_SKIP:
 170          case PIPE_H265_ENC_RATE_CONTROL_METHOD_CONSTANT:
 171             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_CBR;
 172             break;
 173          case PIPE_H265_ENC_RATE_CONTROL_METHOD_VARIABLE_SKIP:
 174          case PIPE_H265_ENC_RATE_CONTROL_METHOD_VARIABLE:
 175             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_PEAK_CONSTRAINED_VBR;
 176             break;
 177          default:
 178             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
 179       }
 180    }
 181 }
 182
 183 static void flush(struct radeon_encoder *enc)
 184 {
 185         enc->ws->cs_flush(enc->cs, PIPE_FLUSH_ASYNC, NULL);
 186 }
 187
 188 static void radeon_enc_flush(struct pipe_video_codec *encoder)
 189 {
 190         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 191         flush(enc);
 192 }
 193
 194 static void radeon_enc_cs_flush(void *ctx, unsigned flags,
 195                                                                 struct pipe_fence_handle **fence)
 196 {
 197         // just ignored
 198 }
 199
 200 static unsigned get_cpb_num(struct radeon_encoder *enc)
 201 {
 202         unsigned w = align(enc->base.width, 16) / 16;
 203         unsigned h = align(enc->base.height, 16) / 16;
 204         unsigned dpb;
 205
 206         switch (enc->base.level) {
 207         case 10:
 208                 dpb = 396;
 209                 break;
 210         case 11:
 211                 dpb = 900;
 212                 break;
 213         case 12:
 214         case 13:
 215         case 20:
 216                 dpb = 2376;
 217                 break;
 218         case 21:
 219                 dpb = 4752;
 220                 break;
 221         case 22:
 222         case 30:
 223                 dpb = 8100;
 224                 break;
 225         case 31:
 226                 dpb = 18000;
 227                 break;
 228         case 32:
 229                 dpb = 20480;
 230                 break;
 231         case 40:
 232         case 41:
 233                 dpb = 32768;
 234                 break;
 235         case 42:
 236                 dpb = 34816;
 237                 break;
 238         case 50:
 239                 dpb = 110400;
 240                 break;
 241         default:
 242         case 51:
 243         case 52:
 244                 dpb = 184320;
 245                 break;
 246         }
 247
 248         return MIN2(dpb / (w * h), 16);
 249 }
 250
 251 static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
 252                                                          struct pipe_video_buffer *source,
 253                                                          struct pipe_picture_desc *picture)
 254 {
 255         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 256         struct vl_video_buffer *vid_buf = (struct vl_video_buffer *)source;
 257
 258         radeon_vcn_enc_get_param(enc, picture);
 259
 260         enc->get_buffer(vid_buf->resources[0], &enc->handle, &enc->luma);
 261         enc->get_buffer(vid_buf->resources[1], NULL, &enc->chroma);
 262
 263         enc->need_feedback = false;
 264
 265         if (!enc->stream_handle) {
 266                 struct rvid_buffer fb;
 267                 enc->stream_handle = si_vid_alloc_stream_handle();
 268                 enc->si = CALLOC_STRUCT(rvid_buffer);
 269                 si_vid_create_buffer(enc->screen, enc->si, 128 * 1024, PIPE_USAGE_STAGING);
 270                 si_vid_create_buffer(enc->screen, &fb, 4096, PIPE_USAGE_STAGING);
 271                 enc->fb = &fb;
 272                 enc->begin(enc);
 273                 flush(enc);
 274                 si_vid_destroy_buffer(&fb);
 275         }
 276 }
 277
 278 static void radeon_enc_encode_bitstream(struct pipe_video_codec *encoder,
 279                                                                   struct pipe_video_buffer *source,
 280                                                                   struct pipe_resource *destination,
 281                                                                   void **fb)
 282 {
 283         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 284         enc->get_buffer(destination, &enc->bs_handle, NULL);
 285         enc->bs_size = destination->width0;
 286
 287         *fb = enc->fb = CALLOC_STRUCT(rvid_buffer);
 288
 289         if (!si_vid_create_buffer(enc->screen, enc->fb, 4096, PIPE_USAGE_STAGING)) {
 290                 RVID_ERR("Can't create feedback buffer.\n");
 291                 return;
 292         }
 293
 294         enc->need_feedback = true;
 295         enc->encode(enc);
 296 }
 297
 298 static void radeon_enc_end_frame(struct pipe_video_codec *encoder,
 299                                                    struct pipe_video_buffer *source,
 300                                                    struct pipe_picture_desc *picture)
 301 {
 302         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 303         flush(enc);
 304 }
 305
 306 static void radeon_enc_destroy(struct pipe_video_codec *encoder)
 307 {
 308         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 309
 310         if (enc->stream_handle) {
 311                 struct rvid_buffer fb;
 312                 enc->need_feedback = false;
 313                 si_vid_create_buffer(enc->screen, &fb, 512, PIPE_USAGE_STAGING);
 314                 enc->fb = &fb;
 315                 enc->destroy(enc);
 316                 flush(enc);
 317                 si_vid_destroy_buffer(&fb);
 318         }
 319
 320         si_vid_destroy_buffer(&enc->cpb);
 321         enc->ws->cs_destroy(enc->cs);
 322         FREE(enc);
 323 }
 324
 325 static void radeon_enc_get_feedback(struct pipe_video_codec *encoder,
 326                                                           void *feedback, unsigned *size)
 327 {
 328         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 329         struct rvid_buffer *fb = feedback;
 330
 331         if (size) {
 332                 uint32_t *ptr = enc->ws->buffer_map(
 333                         fb->res->buf, enc->cs,
 334                         PIPE_TRANSFER_READ_WRITE | RADEON_TRANSFER_TEMPORARY);
 335                 if (ptr[1])
 336                         *size = ptr[6];
 337                 else
 338                         *size = 0;
 339                 enc->ws->buffer_unmap(fb->res->buf);
 340         }
 341
 342         si_vid_destroy_buffer(fb);
 343         FREE(fb);
 344 }
 345
 346 struct pipe_video_codec *radeon_create_encoder(struct pipe_context *context,
 347                 const struct pipe_video_codec *templ,
 348                 struct radeon_winsys* ws,
 349                 radeon_enc_get_buffer get_buffer)
 350 {
 351         struct si_screen *sscreen = (struct si_screen *)context->screen;
 352         struct si_context *sctx = (struct si_context*)context;
 353         struct radeon_encoder *enc;
 354         struct pipe_video_buffer *tmp_buf, templat = {};
 355         struct radeon_surf *tmp_surf;
 356         unsigned cpb_size;
 357
 358         enc = CALLOC_STRUCT(radeon_encoder);
 359
 360         if (!enc)
 361                 return NULL;
 362
 363         enc->alignment = 256;
 364         enc->base = *templ;
 365         enc->base.context = context;
 366         enc->base.destroy = radeon_enc_destroy;
 367         enc->base.begin_frame = radeon_enc_begin_frame;
 368         enc->base.encode_bitstream = radeon_enc_encode_bitstream;
 369         enc->base.end_frame = radeon_enc_end_frame;
 370         enc->base.flush = radeon_enc_flush;
 371         enc->base.get_feedback = radeon_enc_get_feedback;
 372         enc->get_buffer = get_buffer;
 373         enc->bits_in_shifter = 0;
 374         enc->screen = context->screen;
 375         enc->ws = ws;
 376         enc->cs = ws->cs_create(sctx->ctx, RING_VCN_ENC, radeon_enc_cs_flush,
 377                                 enc, false);
 378
 379         if (!enc->cs) {
 380                 RVID_ERR("Can't get command submission context.\n");
 381                 goto error;
 382         }
 383
 384         struct rvid_buffer si;
 385         si_vid_create_buffer(enc->screen, &si, 128 * 1024, PIPE_USAGE_STAGING);
 386         enc->si = &si;
 387
 388         templat.buffer_format = PIPE_FORMAT_NV12;
 389         templat.chroma_format = PIPE_VIDEO_CHROMA_FORMAT_420;
 390         templat.width = enc->base.width;
 391         templat.height = enc->base.height;
 392         templat.interlaced = false;
 393
 394         if (!(tmp_buf = context->create_video_buffer(context, &templat))) {
 395                 RVID_ERR("Can't create video buffer.\n");
 396                 goto error;
 397         }
 398
 399         enc->cpb_num = get_cpb_num(enc);
 400
 401         if (!enc->cpb_num)
 402                 goto error;
 403
 404         get_buffer(((struct vl_video_buffer *)tmp_buf)->resources[0], NULL, &tmp_surf);
 405
 406         cpb_size = (sscreen->info.chip_class < GFX9) ?
 407                            align(tmp_surf->u.legacy.level[0].nblk_x * tmp_surf->bpe, 128) *
 408                            align(tmp_surf->u.legacy.level[0].nblk_y, 32) :
 409                            align(tmp_surf->u.gfx9.surf_pitch * tmp_surf->bpe, 256) *
 410                            align(tmp_surf->u.gfx9.surf_height, 32);
 411
 412         cpb_size = cpb_size * 3 / 2;
 413         cpb_size = cpb_size * enc->cpb_num;
 414         tmp_buf->destroy(tmp_buf);
 415
 416         if (!si_vid_create_buffer(enc->screen, &enc->cpb, cpb_size, PIPE_USAGE_DEFAULT)) {
 417                 RVID_ERR("Can't create CPB buffer.\n");
 418                 goto error;
 419         }
 420
 421         if (sscreen->info.family <= CHIP_RAVEN2)
 422                 radeon_enc_1_2_init(enc);
 423         else
 424                 radeon_enc_2_0_init(enc);
 425
 426         return &enc->base;
 427
 428 error:
 429         if (enc->cs)
 430                 enc->ws->cs_destroy(enc->cs);
 431
 432         si_vid_destroy_buffer(&enc->cpb);
 433
 434         FREE(enc);
 435         return NULL;
 436 }
 437
 438 void radeon_enc_add_buffer(struct radeon_encoder *enc, struct pb_buffer *buf,
 439                            enum radeon_bo_usage usage, enum radeon_bo_domain domain,
 440                            signed offset)
 441 {
 442         enc->ws->cs_add_buffer(enc->cs, buf, usage | RADEON_USAGE_SYNCHRONIZED,
 443                                domain, 0);
 444         uint64_t addr;
 445         addr = enc->ws->buffer_get_virtual_address(buf);
 446         addr = addr + offset;
 447         RADEON_ENC_CS(addr >> 32);
 448         RADEON_ENC_CS(addr);
 449 }
 450
 451 void radeon_enc_set_emulation_prevention(struct radeon_encoder *enc, bool set)
 452 {
 453         if (set != enc->emulation_prevention) {
 454                 enc->emulation_prevention = set;
 455                 enc->num_zeros = 0;
 456         }
 457 }
 458
 459 void radeon_enc_output_one_byte(struct radeon_encoder *enc, unsigned char byte)
 460 {
 461         if (enc->byte_index == 0)
 462                 enc->cs->current.buf[enc->cs->current.cdw] = 0;
 463         enc->cs->current.buf[enc->cs->current.cdw] |= ((unsigned int)(byte) << index_to_shifts[enc->byte_index]);
 464         enc->byte_index++;
 465
 466         if (enc->byte_index >= 4) {
 467                 enc->byte_index = 0;
 468                 enc->cs->current.cdw++;
 469         }
 470 }
 471
 472 void radeon_enc_emulation_prevention(struct radeon_encoder *enc, unsigned char byte)
 473 {
 474         if(enc->emulation_prevention) {
 475                 if((enc->num_zeros >= 2) && ((byte == 0x00) || (byte == 0x01) || (byte == 0x03))) {
 476             radeon_enc_output_one_byte(enc, 0x03);
 477             enc->bits_output += 8;
 478             enc->num_zeros = 0;
 479         }
 480         enc->num_zeros = (byte == 0 ? (enc->num_zeros + 1) : 0);
 481     }
 482 }
 483
 484 void radeon_enc_code_fixed_bits(struct radeon_encoder *enc, unsigned int value, unsigned int num_bits)
 485 {
 486         unsigned int bits_to_pack = 0;
 487
 488         while(num_bits > 0) {
 489                 unsigned int value_to_pack = value & (0xffffffff >> (32 - num_bits));
 490                 bits_to_pack = num_bits > (32 - enc->bits_in_shifter) ? (32 - enc->bits_in_shifter) : num_bits;
 491
 492                 if (bits_to_pack < num_bits)
 493                         value_to_pack = value_to_pack >> (num_bits - bits_to_pack);
 494
 495                 enc->shifter |= value_to_pack << (32 - enc->bits_in_shifter - bits_to_pack);
 496                 num_bits -= bits_to_pack;
 497                 enc->bits_in_shifter += bits_to_pack;
 498
 499                 while(enc->bits_in_shifter >= 8) {
 500                         unsigned char output_byte = (unsigned char)(enc->shifter >> 24);
 501                         enc->shifter <<= 8;
 502                         radeon_enc_emulation_prevention(enc, output_byte);
 503                         radeon_enc_output_one_byte(enc, output_byte);
 504                         enc->bits_in_shifter -= 8;
 505                         enc->bits_output += 8;
 506                 }
 507         }
 508 }
 509
 510 void radeon_enc_reset(struct radeon_encoder *enc)
 511 {
 512         enc->emulation_prevention = false;
 513         enc->shifter = 0;
 514         enc->bits_in_shifter = 0;
 515         enc->bits_output = 0;
 516         enc->num_zeros = 0;
 517         enc->byte_index = 0;
 518 }
 519
 520 void radeon_enc_byte_align(struct radeon_encoder *enc)
 521 {
 522         unsigned int num_padding_zeros = (32 - enc->bits_in_shifter) % 8;
 523
 524         if (num_padding_zeros > 0)
 525                 radeon_enc_code_fixed_bits(enc, 0, num_padding_zeros);
 526 }
 527
 528 void radeon_enc_flush_headers(struct radeon_encoder *enc)
 529 {
 530         if (enc->bits_in_shifter != 0) {
 531                 unsigned char output_byte = (unsigned char)(enc->shifter >> 24);
 532                 radeon_enc_emulation_prevention(enc, output_byte);
 533                 radeon_enc_output_one_byte(enc, output_byte);
 534                 enc->bits_output += enc->bits_in_shifter;
 535                 enc->shifter = 0;
 536                 enc->bits_in_shifter = 0;
 537                 enc->num_zeros = 0;
 538         }
 539
 540         if (enc->byte_index > 0) {
 541                 enc->cs->current.cdw++;
 542                 enc->byte_index = 0;
 543         }
 544 }
 545
 546 void radeon_enc_code_ue(struct radeon_encoder *enc, unsigned int value)
 547 {
 548         int x = -1;
 549         unsigned int ue_code = value + 1;
 550         value += 1;
 551
 552         while (value) {
 553                 value = (value >> 1);
 554                 x += 1;
 555         }
 556
 557         unsigned int ue_length = (x << 1) + 1;
 558         radeon_enc_code_fixed_bits(enc, ue_code, ue_length);
 559 }
 560
 561 void radeon_enc_code_se(struct radeon_encoder *enc, int value)
 562 {
 563         unsigned int v = 0;
 564
 565         if (value != 0)
 566                 v = (value < 0 ? ((unsigned int)(0 - value) << 1) : (((unsigned int)(value) << 1) - 1));
 567
 568         radeon_enc_code_ue(enc, v);
 569 }