src/gallium/drivers/radeon/radeon_vcn_enc.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2017 Advanced Micro Devices, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include <stdio.h>
  29
  30 #include "pipe/p_video_codec.h"
  31
  32 #include "util/u_video.h"
  33 #include "util/u_memory.h"
  34
  35 #include "vl/vl_video_buffer.h"
  36
  37 #include "radeonsi/si_pipe.h"
  38 #include "radeon_video.h"
  39 #include "radeon_vcn_enc.h"
  40
  41 static const unsigned index_to_shifts[4] = {24, 16, 8, 0};
  42
  43 static void radeon_vcn_enc_get_param(struct radeon_encoder *enc, struct pipe_picture_desc *picture)
  44 {
  45    if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_MPEG4_AVC) {
  46       struct pipe_h264_enc_picture_desc *pic = (struct pipe_h264_enc_picture_desc *)picture;
  47       enc->enc_pic.picture_type = pic->picture_type;
  48       enc->enc_pic.frame_num = pic->frame_num;
  49       enc->enc_pic.pic_order_cnt = pic->pic_order_cnt;
  50       enc->enc_pic.pic_order_cnt_type = pic->pic_order_cnt_type;
  51       enc->enc_pic.ref_idx_l0 = pic->ref_idx_l0;
  52       enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1;
  53       enc->enc_pic.not_referenced = pic->not_referenced;
  54       enc->enc_pic.is_idr = (pic->picture_type == PIPE_H264_ENC_PICTURE_TYPE_IDR);
  55       if (pic->pic_ctrl.enc_frame_cropping_flag) {
  56           enc->enc_pic.crop_left = pic->pic_ctrl.enc_frame_crop_left_offset;
  57           enc->enc_pic.crop_right = pic->pic_ctrl.enc_frame_crop_right_offset;
  58           enc->enc_pic.crop_top = pic->pic_ctrl.enc_frame_crop_top_offset;
  59           enc->enc_pic.crop_bottom = pic->pic_ctrl.enc_frame_crop_bottom_offset;
  60       } else {
  61           enc->enc_pic.crop_left = 0;
  62           enc->enc_pic.crop_right = (align(enc->base.width, 16) - enc->base.width) / 2;
  63           enc->enc_pic.crop_top = 0;
  64           enc->enc_pic.crop_bottom = (align(enc->base.height, 16) - enc->base.height) / 2;
  65       }
  66       enc->enc_pic.rc_layer_init.target_bit_rate = pic->rate_ctrl.target_bitrate;
  67       enc->enc_pic.rc_layer_init.peak_bit_rate = pic->rate_ctrl.peak_bitrate;
  68       enc->enc_pic.rc_layer_init.frame_rate_num = pic->rate_ctrl.frame_rate_num;
  69       enc->enc_pic.rc_layer_init.frame_rate_den = pic->rate_ctrl.frame_rate_den;
  70       enc->enc_pic.rc_layer_init.vbv_buffer_size = pic->rate_ctrl.vbv_buffer_size;
  71       enc->enc_pic.rc_layer_init.avg_target_bits_per_picture = pic->rate_ctrl.target_bits_picture;
  72       enc->enc_pic.rc_layer_init.peak_bits_per_picture_integer = pic->rate_ctrl.peak_bits_picture_integer;
  73       enc->enc_pic.rc_layer_init.peak_bits_per_picture_fractional = pic->rate_ctrl.peak_bits_picture_fraction;
  74       enc->enc_pic.rc_session_init.vbv_buffer_level = pic->rate_ctrl.vbv_buf_lv;
  75       enc->enc_pic.rc_per_pic.qp = pic->quant_i_frames;
  76       enc->enc_pic.rc_per_pic.min_qp_app = 0;
  77       enc->enc_pic.rc_per_pic.max_qp_app = 51;
  78       enc->enc_pic.rc_per_pic.max_au_size = 0;
  79       enc->enc_pic.rc_per_pic.enabled_filler_data = pic->rate_ctrl.fill_data_enable;
  80       enc->enc_pic.rc_per_pic.skip_frame_enable = false;
  81       enc->enc_pic.rc_per_pic.enforce_hrd = pic->rate_ctrl.enforce_hrd;
  82       switch(pic->rate_ctrl.rate_ctrl_method) {
  83          case PIPE_H264_ENC_RATE_CONTROL_METHOD_DISABLE:
  84             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
  85             break;
  86          case PIPE_H264_ENC_RATE_CONTROL_METHOD_CONSTANT_SKIP:
  87          case PIPE_H264_ENC_RATE_CONTROL_METHOD_CONSTANT:
  88             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_CBR;
  89             break;
  90          case PIPE_H264_ENC_RATE_CONTROL_METHOD_VARIABLE_SKIP:
  91          case PIPE_H264_ENC_RATE_CONTROL_METHOD_VARIABLE:
  92             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_PEAK_CONSTRAINED_VBR;
  93             break;
  94          default:
  95             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
  96       }
  97    } else if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_HEVC) {
  98       struct pipe_h265_enc_picture_desc *pic = (struct pipe_h265_enc_picture_desc *)picture;
  99       enc->enc_pic.picture_type = pic->picture_type;
 100       enc->enc_pic.frame_num = pic->frame_num;
 101       enc->enc_pic.pic_order_cnt = pic->pic_order_cnt;
 102       enc->enc_pic.pic_order_cnt_type = pic->pic_order_cnt_type;
 103       enc->enc_pic.ref_idx_l0 = pic->ref_idx_l0;
 104       enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1;
 105       enc->enc_pic.not_referenced = pic->not_referenced;
 106       enc->enc_pic.is_idr = (pic->picture_type == PIPE_H265_ENC_PICTURE_TYPE_IDR) ||
 107                             (pic->picture_type == PIPE_H265_ENC_PICTURE_TYPE_I);
 108       enc->enc_pic.crop_left = 0;
 109       enc->enc_pic.crop_right = (align(enc->base.width, 16) - enc->base.width) / 2;
 110       enc->enc_pic.crop_top = 0;
 111       enc->enc_pic.crop_bottom = (align(enc->base.height, 16) - enc->base.height) / 2;
 112       enc->enc_pic.general_tier_flag = pic->seq.general_tier_flag;
 113       enc->enc_pic.general_profile_idc = pic->seq.general_profile_idc;
 114       enc->enc_pic.general_level_idc = pic->seq.general_level_idc;
 115       enc->enc_pic.max_poc =
 116          MAX2(16, util_next_power_of_two(pic->seq.intra_period));
 117       enc->enc_pic.log2_max_poc = 0;
 118       for (int i = enc->enc_pic.max_poc; i != 0; enc->enc_pic.log2_max_poc++)
 119          i = (i >> 1);
 120       enc->enc_pic.chroma_format_idc = pic->seq.chroma_format_idc;
 121       enc->enc_pic.pic_width_in_luma_samples = pic->seq.pic_width_in_luma_samples;
 122       enc->enc_pic.pic_height_in_luma_samples = pic->seq.pic_height_in_luma_samples;
 123       enc->enc_pic.log2_diff_max_min_luma_coding_block_size = pic->seq.log2_diff_max_min_luma_coding_block_size;
 124       enc->enc_pic.log2_min_transform_block_size_minus2 = pic->seq.log2_min_transform_block_size_minus2;
 125       enc->enc_pic.log2_diff_max_min_transform_block_size = pic->seq.log2_diff_max_min_transform_block_size;
 126       enc->enc_pic.max_transform_hierarchy_depth_inter = pic->seq.max_transform_hierarchy_depth_inter;
 127       enc->enc_pic.max_transform_hierarchy_depth_intra = pic->seq.max_transform_hierarchy_depth_intra;
 128       enc->enc_pic.log2_parallel_merge_level_minus2 = pic->pic.log2_parallel_merge_level_minus2;
 129       enc->enc_pic.bit_depth_luma_minus8 = pic->seq.bit_depth_luma_minus8;
 130       enc->enc_pic.bit_depth_chroma_minus8 = pic->seq.bit_depth_chroma_minus8;
 131       enc->enc_pic.nal_unit_type = pic->pic.nal_unit_type;
 132       enc->enc_pic.max_num_merge_cand = pic->slice.max_num_merge_cand;
 133       enc->enc_pic.sample_adaptive_offset_enabled_flag = pic->seq.sample_adaptive_offset_enabled_flag;
 134       enc->enc_pic.pcm_enabled_flag = pic->seq.pcm_enabled_flag;
 135       enc->enc_pic.sps_temporal_mvp_enabled_flag = pic->seq.sps_temporal_mvp_enabled_flag;
 136       enc->enc_pic.hevc_deblock.loop_filter_across_slices_enabled = pic->slice.slice_loop_filter_across_slices_enabled_flag;
 137       enc->enc_pic.hevc_deblock.deblocking_filter_disabled = pic->slice.slice_deblocking_filter_disabled_flag;
 138       enc->enc_pic.hevc_deblock.beta_offset_div2 = pic->slice.slice_beta_offset_div2;
 139       enc->enc_pic.hevc_deblock.tc_offset_div2 = pic->slice.slice_tc_offset_div2;
 140       enc->enc_pic.hevc_deblock.cb_qp_offset = pic->slice.slice_cb_qp_offset;
 141       enc->enc_pic.hevc_deblock.cr_qp_offset = pic->slice.slice_cr_qp_offset;
 142       enc->enc_pic.hevc_spec_misc.log2_min_luma_coding_block_size_minus3 = pic->seq.log2_min_luma_coding_block_size_minus3;
 143       enc->enc_pic.hevc_spec_misc.amp_disabled = !pic->seq.amp_enabled_flag;
 144       enc->enc_pic.hevc_spec_misc.strong_intra_smoothing_enabled = pic->seq.strong_intra_smoothing_enabled_flag;
 145       enc->enc_pic.hevc_spec_misc.constrained_intra_pred_flag = pic->pic.constrained_intra_pred_flag;
 146       enc->enc_pic.hevc_spec_misc.cabac_init_flag = pic->slice.cabac_init_flag;
 147       enc->enc_pic.hevc_spec_misc.half_pel_enabled = 1;
 148       enc->enc_pic.hevc_spec_misc.quarter_pel_enabled = 1;
 149       enc->enc_pic.rc_layer_init.target_bit_rate = pic->rc.target_bitrate;
 150       enc->enc_pic.rc_layer_init.peak_bit_rate = pic->rc.peak_bitrate;
 151       enc->enc_pic.rc_layer_init.frame_rate_num = pic->rc.frame_rate_num;
 152       enc->enc_pic.rc_layer_init.frame_rate_den = pic->rc.frame_rate_den;
 153       enc->enc_pic.rc_layer_init.vbv_buffer_size = pic->rc.vbv_buffer_size;
 154       enc->enc_pic.rc_layer_init.avg_target_bits_per_picture = pic->rc.target_bits_picture;
 155       enc->enc_pic.rc_layer_init.peak_bits_per_picture_integer = pic->rc.peak_bits_picture_integer;
 156       enc->enc_pic.rc_layer_init.peak_bits_per_picture_fractional = pic->rc.peak_bits_picture_fraction;
 157       enc->enc_pic.rc_session_init.vbv_buffer_level = pic->rc.vbv_buf_lv;
 158       enc->enc_pic.rc_per_pic.qp = pic->rc.quant_i_frames;
 159       enc->enc_pic.rc_per_pic.min_qp_app = 0;
 160       enc->enc_pic.rc_per_pic.max_qp_app = 51;
 161       enc->enc_pic.rc_per_pic.max_au_size = 0;
 162       enc->enc_pic.rc_per_pic.enabled_filler_data = pic->rc.fill_data_enable;
 163       enc->enc_pic.rc_per_pic.skip_frame_enable = false;
 164       enc->enc_pic.rc_per_pic.enforce_hrd = pic->rc.enforce_hrd;
 165       switch(pic->rc.rate_ctrl_method) {
 166          case PIPE_H265_ENC_RATE_CONTROL_METHOD_DISABLE:
 167          enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
 168             break;
 169          case PIPE_H265_ENC_RATE_CONTROL_METHOD_CONSTANT_SKIP:
 170          case PIPE_H265_ENC_RATE_CONTROL_METHOD_CONSTANT:
 171             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_CBR;
 172             break;
 173          case PIPE_H265_ENC_RATE_CONTROL_METHOD_VARIABLE_SKIP:
 174          case PIPE_H265_ENC_RATE_CONTROL_METHOD_VARIABLE:
 175             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_PEAK_CONSTRAINED_VBR;
 176             break;
 177          default:
 178             enc->enc_pic.rc_session_init.rate_control_method = RENCODE_RATE_CONTROL_METHOD_NONE;
 179       }
 180    }
 181 }
 182
 183 static void flush(struct radeon_encoder *enc)
 184 {
 185         enc->ws->cs_flush(enc->cs, PIPE_FLUSH_ASYNC, NULL);
 186 }
 187
 188 static void radeon_enc_flush(struct pipe_video_codec *encoder)
 189 {
 190         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 191         flush(enc);
 192 }
 193
 194 static void radeon_enc_cs_flush(void *ctx, unsigned flags,
 195                                                                 struct pipe_fence_handle **fence)
 196 {
 197         // just ignored
 198 }
 199
 200 static unsigned get_cpb_num(struct radeon_encoder *enc)
 201 {
 202         unsigned w = align(enc->base.width, 16) / 16;
 203         unsigned h = align(enc->base.height, 16) / 16;
 204         unsigned dpb;
 205
 206         switch (enc->base.level) {
 207         case 10:
 208                 dpb = 396;
 209                 break;
 210         case 11:
 211                 dpb = 900;
 212                 break;
 213         case 12:
 214         case 13:
 215         case 20:
 216                 dpb = 2376;
 217                 break;
 218         case 21:
 219                 dpb = 4752;
 220                 break;
 221         case 22:
 222         case 30:
 223                 dpb = 8100;
 224                 break;
 225         case 31:
 226                 dpb = 18000;
 227                 break;
 228         case 32:
 229                 dpb = 20480;
 230                 break;
 231         case 40:
 232         case 41:
 233                 dpb = 32768;
 234                 break;
 235         case 42:
 236                 dpb = 34816;
 237                 break;
 238         case 50:
 239                 dpb = 110400;
 240                 break;
 241         default:
 242         case 51:
 243         case 52:
 244                 dpb = 184320;
 245                 break;
 246         }
 247
 248         return MIN2(dpb / (w * h), 16);
 249 }
 250
 251 static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
 252                                                          struct pipe_video_buffer *source,
 253                                                          struct pipe_picture_desc *picture)
 254 {
 255         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 256         struct vl_video_buffer *vid_buf = (struct vl_video_buffer *)source;
 257         bool need_rate_control = false;
 258
 259         if (u_reduce_video_profile(enc->base.profile) == PIPE_VIDEO_FORMAT_MPEG4_AVC) {
 260                 struct pipe_h264_enc_picture_desc *pic = (struct pipe_h264_enc_picture_desc *)picture;
 261                 need_rate_control =
 262                         enc->enc_pic.rc_layer_init.target_bit_rate != pic->rate_ctrl.target_bitrate;
 263         } else if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_HEVC) {
 264                 struct pipe_h265_enc_picture_desc *pic = (struct pipe_h265_enc_picture_desc *)picture;
 265                 need_rate_control =
 266                         enc->enc_pic.rc_layer_init.target_bit_rate != pic->rc.target_bitrate;
 267         }
 268
 269         radeon_vcn_enc_get_param(enc, picture);
 270
 271         enc->get_buffer(vid_buf->resources[0], &enc->handle, &enc->luma);
 272         enc->get_buffer(vid_buf->resources[1], NULL, &enc->chroma);
 273
 274         enc->need_feedback = false;
 275
 276         if (!enc->stream_handle) {
 277                 struct rvid_buffer fb;
 278                 enc->stream_handle = si_vid_alloc_stream_handle();
 279                 enc->si = CALLOC_STRUCT(rvid_buffer);
 280                 si_vid_create_buffer(enc->screen, enc->si, 128 * 1024, PIPE_USAGE_STAGING);
 281                 si_vid_create_buffer(enc->screen, &fb, 4096, PIPE_USAGE_STAGING);
 282                 enc->fb = &fb;
 283                 enc->begin(enc);
 284                 flush(enc);
 285                 si_vid_destroy_buffer(&fb);
 286         }
 287         if (need_rate_control) {
 288                 enc->begin(enc);
 289                 flush(enc);
 290         }
 291 }
 292
 293 static void radeon_enc_encode_bitstream(struct pipe_video_codec *encoder,
 294                                                                   struct pipe_video_buffer *source,
 295                                                                   struct pipe_resource *destination,
 296                                                                   void **fb)
 297 {
 298         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 299         enc->get_buffer(destination, &enc->bs_handle, NULL);
 300         enc->bs_size = destination->width0;
 301
 302         *fb = enc->fb = CALLOC_STRUCT(rvid_buffer);
 303
 304         if (!si_vid_create_buffer(enc->screen, enc->fb, 4096, PIPE_USAGE_STAGING)) {
 305                 RVID_ERR("Can't create feedback buffer.\n");
 306                 return;
 307         }
 308
 309         enc->need_feedback = true;
 310         enc->encode(enc);
 311 }
 312
 313 static void radeon_enc_end_frame(struct pipe_video_codec *encoder,
 314                                                    struct pipe_video_buffer *source,
 315                                                    struct pipe_picture_desc *picture)
 316 {
 317         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 318         flush(enc);
 319 }
 320
 321 static void radeon_enc_destroy(struct pipe_video_codec *encoder)
 322 {
 323         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 324
 325         if (enc->stream_handle) {
 326                 struct rvid_buffer fb;
 327                 enc->need_feedback = false;
 328                 si_vid_create_buffer(enc->screen, &fb, 512, PIPE_USAGE_STAGING);
 329                 enc->fb = &fb;
 330                 enc->destroy(enc);
 331                 flush(enc);
 332                 si_vid_destroy_buffer(&fb);
 333         }
 334
 335         si_vid_destroy_buffer(&enc->cpb);
 336         enc->ws->cs_destroy(enc->cs);
 337         FREE(enc);
 338 }
 339
 340 static void radeon_enc_get_feedback(struct pipe_video_codec *encoder,
 341                                                           void *feedback, unsigned *size)
 342 {
 343         struct radeon_encoder *enc = (struct radeon_encoder*)encoder;
 344         struct rvid_buffer *fb = feedback;
 345
 346         if (size) {
 347                 uint32_t *ptr = enc->ws->buffer_map(
 348                         fb->res->buf, enc->cs,
 349                         PIPE_TRANSFER_READ_WRITE | RADEON_TRANSFER_TEMPORARY);
 350                 if (ptr[1])
 351                         *size = ptr[6];
 352                 else
 353                         *size = 0;
 354                 enc->ws->buffer_unmap(fb->res->buf);
 355         }
 356
 357         si_vid_destroy_buffer(fb);
 358         FREE(fb);
 359 }
 360
 361 struct pipe_video_codec *radeon_create_encoder(struct pipe_context *context,
 362                 const struct pipe_video_codec *templ,
 363                 struct radeon_winsys* ws,
 364                 radeon_enc_get_buffer get_buffer)
 365 {
 366         struct si_screen *sscreen = (struct si_screen *)context->screen;
 367         struct si_context *sctx = (struct si_context*)context;
 368         struct radeon_encoder *enc;
 369         struct pipe_video_buffer *tmp_buf, templat = {};
 370         struct radeon_surf *tmp_surf;
 371         unsigned cpb_size;
 372
 373         enc = CALLOC_STRUCT(radeon_encoder);
 374
 375         if (!enc)
 376                 return NULL;
 377
 378         enc->alignment = 256;
 379         enc->base = *templ;
 380         enc->base.context = context;
 381         enc->base.destroy = radeon_enc_destroy;
 382         enc->base.begin_frame = radeon_enc_begin_frame;
 383         enc->base.encode_bitstream = radeon_enc_encode_bitstream;
 384         enc->base.end_frame = radeon_enc_end_frame;
 385         enc->base.flush = radeon_enc_flush;
 386         enc->base.get_feedback = radeon_enc_get_feedback;
 387         enc->get_buffer = get_buffer;
 388         enc->bits_in_shifter = 0;
 389         enc->screen = context->screen;
 390         enc->ws = ws;
 391         enc->cs = ws->cs_create(sctx->ctx, RING_VCN_ENC, radeon_enc_cs_flush,
 392                                 enc, false);
 393
 394         if (!enc->cs) {
 395                 RVID_ERR("Can't get command submission context.\n");
 396                 goto error;
 397         }
 398
 399         struct rvid_buffer si;
 400         si_vid_create_buffer(enc->screen, &si, 128 * 1024, PIPE_USAGE_STAGING);
 401         enc->si = &si;
 402
 403         templat.buffer_format = PIPE_FORMAT_NV12;
 404         templat.chroma_format = PIPE_VIDEO_CHROMA_FORMAT_420;
 405         templat.width = enc->base.width;
 406         templat.height = enc->base.height;
 407         templat.interlaced = false;
 408
 409         if (!(tmp_buf = context->create_video_buffer(context, &templat))) {
 410                 RVID_ERR("Can't create video buffer.\n");
 411                 goto error;
 412         }
 413
 414         enc->cpb_num = get_cpb_num(enc);
 415
 416         if (!enc->cpb_num)
 417                 goto error;
 418
 419         get_buffer(((struct vl_video_buffer *)tmp_buf)->resources[0], NULL, &tmp_surf);
 420
 421         cpb_size = (sscreen->info.chip_class < GFX9) ?
 422                            align(tmp_surf->u.legacy.level[0].nblk_x * tmp_surf->bpe, 128) *
 423                            align(tmp_surf->u.legacy.level[0].nblk_y, 32) :
 424                            align(tmp_surf->u.gfx9.surf_pitch * tmp_surf->bpe, 256) *
 425                            align(tmp_surf->u.gfx9.surf_height, 32);
 426
 427         cpb_size = cpb_size * 3 / 2;
 428         cpb_size = cpb_size * enc->cpb_num;
 429         tmp_buf->destroy(tmp_buf);
 430
 431         if (!si_vid_create_buffer(enc->screen, &enc->cpb, cpb_size, PIPE_USAGE_DEFAULT)) {
 432                 RVID_ERR("Can't create CPB buffer.\n");
 433                 goto error;
 434         }
 435
 436         if (sscreen->info.family <= CHIP_RAVEN2)
 437                 radeon_enc_1_2_init(enc);
 438         else
 439                 radeon_enc_2_0_init(enc);
 440
 441         return &enc->base;
 442
 443 error:
 444         if (enc->cs)
 445                 enc->ws->cs_destroy(enc->cs);
 446
 447         si_vid_destroy_buffer(&enc->cpb);
 448
 449         FREE(enc);
 450         return NULL;
 451 }
 452
 453 void radeon_enc_add_buffer(struct radeon_encoder *enc, struct pb_buffer *buf,
 454                            enum radeon_bo_usage usage, enum radeon_bo_domain domain,
 455                            signed offset)
 456 {
 457         enc->ws->cs_add_buffer(enc->cs, buf, usage | RADEON_USAGE_SYNCHRONIZED,
 458                                domain, 0);
 459         uint64_t addr;
 460         addr = enc->ws->buffer_get_virtual_address(buf);
 461         addr = addr + offset;
 462         RADEON_ENC_CS(addr >> 32);
 463         RADEON_ENC_CS(addr);
 464 }
 465
 466 void radeon_enc_set_emulation_prevention(struct radeon_encoder *enc, bool set)
 467 {
 468         if (set != enc->emulation_prevention) {
 469                 enc->emulation_prevention = set;
 470                 enc->num_zeros = 0;
 471         }
 472 }
 473
 474 void radeon_enc_output_one_byte(struct radeon_encoder *enc, unsigned char byte)
 475 {
 476         if (enc->byte_index == 0)
 477                 enc->cs->current.buf[enc->cs->current.cdw] = 0;
 478         enc->cs->current.buf[enc->cs->current.cdw] |= ((unsigned int)(byte) << index_to_shifts[enc->byte_index]);
 479         enc->byte_index++;
 480
 481         if (enc->byte_index >= 4) {
 482                 enc->byte_index = 0;
 483                 enc->cs->current.cdw++;
 484         }
 485 }
 486
 487 void radeon_enc_emulation_prevention(struct radeon_encoder *enc, unsigned char byte)
 488 {
 489         if(enc->emulation_prevention) {
 490                 if((enc->num_zeros >= 2) && ((byte == 0x00) || (byte == 0x01) || (byte == 0x03))) {
 491             radeon_enc_output_one_byte(enc, 0x03);
 492             enc->bits_output += 8;
 493             enc->num_zeros = 0;
 494         }
 495         enc->num_zeros = (byte == 0 ? (enc->num_zeros + 1) : 0);
 496     }
 497 }
 498
 499 void radeon_enc_code_fixed_bits(struct radeon_encoder *enc, unsigned int value, unsigned int num_bits)
 500 {
 501         unsigned int bits_to_pack = 0;
 502
 503         while(num_bits > 0) {
 504                 unsigned int value_to_pack = value & (0xffffffff >> (32 - num_bits));
 505                 bits_to_pack = num_bits > (32 - enc->bits_in_shifter) ? (32 - enc->bits_in_shifter) : num_bits;
 506
 507                 if (bits_to_pack < num_bits)
 508                         value_to_pack = value_to_pack >> (num_bits - bits_to_pack);
 509
 510                 enc->shifter |= value_to_pack << (32 - enc->bits_in_shifter - bits_to_pack);
 511                 num_bits -= bits_to_pack;
 512                 enc->bits_in_shifter += bits_to_pack;
 513
 514                 while(enc->bits_in_shifter >= 8) {
 515                         unsigned char output_byte = (unsigned char)(enc->shifter >> 24);
 516                         enc->shifter <<= 8;
 517                         radeon_enc_emulation_prevention(enc, output_byte);
 518                         radeon_enc_output_one_byte(enc, output_byte);
 519                         enc->bits_in_shifter -= 8;
 520                         enc->bits_output += 8;
 521                 }
 522         }
 523 }
 524
 525 void radeon_enc_reset(struct radeon_encoder *enc)
 526 {
 527         enc->emulation_prevention = false;
 528         enc->shifter = 0;
 529         enc->bits_in_shifter = 0;
 530         enc->bits_output = 0;
 531         enc->num_zeros = 0;
 532         enc->byte_index = 0;
 533 }
 534
 535 void radeon_enc_byte_align(struct radeon_encoder *enc)
 536 {
 537         unsigned int num_padding_zeros = (32 - enc->bits_in_shifter) % 8;
 538
 539         if (num_padding_zeros > 0)
 540                 radeon_enc_code_fixed_bits(enc, 0, num_padding_zeros);
 541 }
 542
 543 void radeon_enc_flush_headers(struct radeon_encoder *enc)
 544 {
 545         if (enc->bits_in_shifter != 0) {
 546                 unsigned char output_byte = (unsigned char)(enc->shifter >> 24);
 547                 radeon_enc_emulation_prevention(enc, output_byte);
 548                 radeon_enc_output_one_byte(enc, output_byte);
 549                 enc->bits_output += enc->bits_in_shifter;
 550                 enc->shifter = 0;
 551                 enc->bits_in_shifter = 0;
 552                 enc->num_zeros = 0;
 553         }
 554
 555         if (enc->byte_index > 0) {
 556                 enc->cs->current.cdw++;
 557                 enc->byte_index = 0;
 558         }
 559 }
 560
 561 void radeon_enc_code_ue(struct radeon_encoder *enc, unsigned int value)
 562 {
 563         int x = -1;
 564         unsigned int ue_code = value + 1;
 565         value += 1;
 566
 567         while (value) {
 568                 value = (value >> 1);
 569                 x += 1;
 570         }
 571
 572         unsigned int ue_length = (x << 1) + 1;
 573         radeon_enc_code_fixed_bits(enc, ue_code, ue_length);
 574 }
 575
 576 void radeon_enc_code_se(struct radeon_encoder *enc, int value)
 577 {
 578         unsigned int v = 0;
 579
 580         if (value != 0)
 581                 v = (value < 0 ? ((unsigned int)(0 - value) << 1) : (((unsigned int)(value) << 1) - 1));
 582
 583         radeon_enc_code_ue(enc, v);
 584 }