obs-studio 数据采集 渲染 编码过程

程序启动时 会调用 obs_init_video函数，创建一个obs_video_thread 线程

1. static int obs_init_video(struct obs_video_info *ovi)

2. {

3. struct obs_core_video *video = &obs->video;

4. struct video_output_info vi;

5. int errorcode;

6.  

7. make_video_info(&vi, ovi);

8. video->base_width = ovi->base_width;

9. video->base_height = ovi->base_height;

10. video->output_width = ovi->output_width;

11. video->output_height = ovi->output_height;

12. video->gpu_conversion = ovi->gpu_conversion;

13. video->scale_type = ovi->scale_type;

14.  

15. set_video_matrix(video, ovi);

16.  

17. errorcode = video_output_open(&video->video, &vi);

18.  

19. if (errorcode != VIDEO_OUTPUT_SUCCESS) {

20. if (errorcode == VIDEO_OUTPUT_INVALIDPARAM) {

21. blog(LOG_ERROR, "Invalid video parameters specified");

22. return OBS_VIDEO_INVALID_PARAM;

23. } else {

24. blog(LOG_ERROR, "Could not open video output");

25. }

26. return OBS_VIDEO_FAIL;

27. }

28.  

29. gs_enter_context(video->graphics);

30.  

31. if (ovi->gpu_conversion && !obs_init_gpu_conversion(ovi))

32. return OBS_VIDEO_FAIL;

33. if (!obs_init_textures(ovi))

34. return OBS_VIDEO_FAIL;

35.  

36. gs_leave_context();

37.  

38. errorcode = pthread_create(&video->video_thread, NULL,

39. obs_video_thread, obs);

40. if (errorcode != 0)

41. return OBS_VIDEO_FAIL;

42.  

43. video->thread_initialized = true;

44. return OBS_VIDEO_SUCCESS;

45. }

obs_video_thread线程中 进行 数据采集 ，渲染 、保存数据到 缓冲区

 

1. void *obs_video_thread(void *param)

2. {

3. uint64_t last_time = 0;

4. uint64_t interval = video_output_get_frame_time(obs->video.video);

5. uint64_t fps_total_ns = 0;

6. uint32_t fps_total_frames = 0;

7.  

8. obs->video.video_time = os_gettime_ns();

9.  

10. os_set_thread_name("libobs: graphics thread");

11.  

12. const char *video_thread_name =

13. profile_store_name(obs_get_profiler_name_store(),

14. "obs_video_thread(%g"NBSP"ms)", interval / 1000000.);

15. profile_register_root(video_thread_name, interval);

16.  

17. while (!video_output_stopped(obs->video.video)) {

18. profile_start(video_thread_name);

19.  

20. profile_start(tick_sources_name);

21. last_time = tick_sources(obs->video.video_time, last_time);

22. profile_end(tick_sources_name);

23.  

24. profile_start(render_displays_name);

25. render_displays();

26. profile_end(render_displays_name);

27.  

28. profile_start(output_frame_name);

29. output_frame();

30. profile_end(output_frame_name);

31.  

32. profile_end(video_thread_name);

33.  

34. profile_reenable_thread();

35.  

36. video_sleep(&obs->video, &obs->video.video_time, interval);

37.  

38. fps_total_ns += (obs->video.video_time - last_time);

39. fps_total_frames++;

40.  

41. if (fps_total_ns >= 1000000000ULL) {

42. obs->video.video_fps = (double)fps_total_frames /

43. ((double)fps_total_ns / 1000000000.0);

44. fps_total_ns = 0;

45. fps_total_frames = 0;

46. }

47. }

48.  

49. UNUSED_PARAMETER(param);

50. return NULL;

51. }

tick_sources遍历当前加入的所有source，调用obs_source_video_tick

 

1. static uint64_t tick_sources(uint64_t cur_time, uint64_t last_time)

2. {

3. struct obs_core_data *data = &obs->data;

4. struct obs_source *source;

5. uint64_t delta_time;

6. float seconds;

7.  

8. if (!last_time)

9. last_time = cur_time -

10. video_output_get_frame_time(obs->video.video);

11.  

12. delta_time = cur_time - last_time;

13. seconds = (float)((double)delta_time / 1000000000.0);

14.  

15. pthread_mutex_lock(&data->sources_mutex);

16.  

17. /* call the tick function of each source */

18. source = data->first_source;

19. while (source) {

20. obs_source_video_tick(source, seconds);

21. source = (struct obs_source*)source->context.next;

22. }

23.  

24. pthread_mutex_unlock(&data->sources_mutex);

25.  

26. return cur_time;

27. }

    obs_source_video_tick 最终会调用 函数指针 video_tick 

1. void obs_source_video_tick(obs_source_t *source, float seconds)

2. {

3. bool now_showing, now_active;

4.  

5. if (!obs_source_valid(source, "obs_source_video_tick"))

6. return;

7.  

8. if (source->info.type == OBS_SOURCE_TYPE_TRANSITION)

9. obs_transition_tick(source);

10.  

11. if ((source->info.output_flags & OBS_SOURCE_ASYNC) != 0) {

12. uint64_t sys_time = obs->video.video_time;

13.  

14. pthread_mutex_lock(&source->async_mutex);

15.  

16. if (deinterlacing_enabled(source)) {

17. deinterlace_process_last_frame(source, sys_time);

18. } else {

19. if (source->cur_async_frame) {

20. remove_async_frame(source,

21. source->cur_async_frame);

22. source->cur_async_frame = NULL;

23. }

24.  

25. source->cur_async_frame = get_closest_frame(source,

26. sys_time);

27. }

28.  

29. source->last_sys_timestamp = sys_time;

30. pthread_mutex_unlock(&source->async_mutex);

31. }

32.  

33. if (source->defer_update)

34. obs_source_deferred_update(source);

35.  

36. /* reset the filter render texture information once every frame */

37. if (source->filter_texrender)

38. gs_texrender_reset(source->filter_texrender);

39.  

40. /* call show/hide if the reference changed */

41. now_showing = !!source->show_refs;

42. if (now_showing != source->showing) {

43. if (now_showing) {

44. show_source(source);

45. } else {

46. hide_source(source);

47. }

48.  

49. source->showing = now_showing;

50. }

51.  

52. /* call activate/deactivate if the reference changed */

53. now_active = !!source->activate_refs;

54. if (now_active != source->active) {

55. if (now_active) {

56. activate_source(source);

57. } else {

58. deactivate_source(source);

59. }

60.  

61. source->active = now_active;

62. }

63.  

64. if (source->context.data && source->info.video_tick)

65. source->info.video_tick(source->context.data, seconds);

66.  

67. source->async_rendered = false;

68. source->deinterlace_rendered = false;

69. }

   video_tick 在每个插件对应的结构体中进行初始化，例如window-capture 窗口捕获

 

1. struct obs_source_info window_capture_info = {

2. .id = "window_capture",

3. .type = OBS_SOURCE_TYPE_INPUT,

4. .output_flags = OBS_SOURCE_VIDEO | OBS_SOURCE_CUSTOM_DRAW,

5. .get_name = wc_getname,

6. .create = wc_create,

7. .destroy = wc_destroy,

8. .update = wc_update,

9. .video_render = wc_render,

10. .video_tick = wc_tick,

11. .get_width = wc_width,

12. .get_height = wc_height,

13. .get_defaults = wc_defaults,

14. .get_properties = wc_properties

15. };

wc_tick进行数据采集，采集的数据放到了source->context.data中 数据读取到后 进行渲染

 

1. static inline void render_displays(void)

2. {

3. struct obs_display *display;

4.  

5. if (!obs->data.valid)

6. return;

7.  

8. gs_enter_context(obs->video.graphics);

9.  

10. /* render extra displays/swaps */

11. pthread_mutex_lock(&obs->data.displays_mutex);

12.  

13. display = obs->data.first_display;

14. while (display) {

15. render_display(display);

16. display = display->next;

17. }

18.  

19. pthread_mutex_unlock(&obs->data.displays_mutex);

20.  

21. gs_leave_context();

22. }

 

1. void render_display(struct obs_display *display)

2. {

3. if (!display || !display->enabled) return;

4.  

5. render_display_begin(display);

6.  

7. pthread_mutex_lock(&display->draw_callbacks_mutex);

8.  

9. for (size_t i = 0; i < display->draw_callbacks.num; i++) {

10. struct draw_callback *callback;

11. callback = display->draw_callbacks.array+i;

12.  

13. callback->draw(callback->param, display->cx, display->cy);

14. }

15.  

16. pthread_mutex_unlock(&display->draw_callbacks_mutex);

17.  

18. render_display_end();

19. }

callback->draw 对应 RenderMain 最终

1. void obs_view_render(obs_view_t *view)

2. {

3. if (!view) return;

4.  

5. pthread_mutex_lock(&view->channels_mutex);

6.  

7. for (size_t i = 0; i < MAX_CHANNELS; i++) {

8. struct obs_source *source;

9.  

10. source = view->channels[i];

11.  

12. if (source) {

13. if (source->removed) {

14. obs_source_release(source);

15. view->channels[i] = NULL;

16. } else {

17. obs_source_video_render(source);

18. }

19. }

20. }

21.  

22. pthread_mutex_unlock(&view->channels_mutex);

23. }

会调用到

 

1. static inline void obs_source_main_render(obs_source_t *source)

2. {

3. uint32_t flags = source->info.output_flags;

4. bool custom_draw = (flags & OBS_SOURCE_CUSTOM_DRAW) != 0;

5. bool default_effect = !source->filter_parent &&

6. source->filters.num == 0 &&

7. !custom_draw;

8.  

9. if (default_effect)

10. obs_source_default_render(source);

11. else if (source->context.data)

12. source->info.video_render(source->context.data,

13. custom_draw ? NULL : gs_get_effect());

14. }

而video_render在插件结构体中进行了初始化，最后调用到wc_render，然后数据传送到 opengl  或者d3d中进行处理，显示

 

1. static void wc_render(void *data, gs_effect_t *effect)

2. {

3. struct window_capture *wc = data;

4. dc_capture_render(&wc->capture, obs_get_base_effect(OBS_EFFECT_OPAQUE));

5.  

6. UNUSED_PARAMETER(effect);

7. }

而obs_video_thread线程通过out_frame获取opengl 或者d3d处理后的数据，然后把数据放到缓冲区

 

1. static inline void output_frame(void)

2. {

3. struct obs_core_video *video = &obs->video;

4. int cur_texture = video->cur_texture;

5. int prev_texture = cur_texture == 0 ? NUM_TEXTURES-1 : cur_texture-1;

6. struct video_data frame;

7. bool frame_ready;

8.  

9. memset(&frame, 0, sizeof(struct video_data));

10.  

11. profile_start(output_frame_gs_context_name);

12. gs_enter_context(video->graphics);

13.  

14. profile_start(output_frame_render_video_name);

15. render_video(video, cur_texture, prev_texture);

16. profile_end(output_frame_render_video_name);

17.  

18. profile_start(output_frame_download_frame_name);

19. frame_ready = download_frame(video, prev_texture, &frame);//获得数据buffer指针

20. profile_end(output_frame_download_frame_name);

21.  

22. profile_start(output_frame_gs_flush_name);

23. gs_flush();

24. profile_end(output_frame_gs_flush_name);

25.  

26. gs_leave_context();

27. profile_end(output_frame_gs_context_name);

28.  

29. if (frame_ready) {

30. struct obs_vframe_info vframe_info;

31. circlebuf_pop_front(&video->vframe_info_buffer, &vframe_info,

32. sizeof(vframe_info));

33.  

34. frame.timestamp = vframe_info.timestamp;

35. profile_start(output_frame_output_video_data_name);

36. output_video_data(video, &frame, vframe_info.count);

37. profile_end(output_frame_output_video_data_name);

38. }

39.  

40. if (++video->cur_texture == NUM_TEXTURES)

41. video->cur_texture = 0;

42. }

数据最终保存在video cache中，

 

1. bool video_output_lock_frame(video_t *video, struct video_frame *frame,

2. int count, uint64_t timestamp)

3. {

4. struct cached_frame_info *cfi;

5. bool locked;

6.  

7. if (!video) return false;

8.  

9. pthread_mutex_lock(&video->data_mutex);

10.  

11. if (video->available_frames == 0) {

12. video->skipped_frames += count;

13. video->cache[video->last_added].count += count;

14. locked = false;

15.  

16. } else {

17. if (video->available_frames != video->info.cache_size) {

18. if (++video->last_added == video->info.cache_size)

19. video->last_added = 0;

20. }

21.  

22. cfi = &video->cache[video->last_added];

23. cfi->frame.timestamp = timestamp;

24. cfi->count = count;

25.  

26. memcpy(frame, &cfi->frame, sizeof(*frame));

27.  

28. locked = true;

29. }

30.  

31. pthread_mutex_unlock(&video->data_mutex);

32.  

33. return locked;

34. }

1. static inline void video_sleep(struct obs_core_video *video,

2. uint64_t *p_time, uint64_t interval_ns)

3. {

4. struct obs_vframe_info vframe_info;

5. uint64_t cur_time = *p_time;

6. uint64_t t = cur_time + interval_ns;

7. int count;

8.  

9. if (os_sleepto_ns(t)) {

10. *p_time = t;

11. count = 1;

12. } else {

13. count = (int)((os_gettime_ns() - cur_time) / interval_ns);

14. *p_time = cur_time + interval_ns * count;

15. }

16.  

17. video->total_frames += count;

18. video->lagged_frames += count - 1;

19.  

20. vframe_info.timestamp = cur_time;

21. vframe_info.count = count;

22. circlebuf_push_back(&video->vframe_info_buffer, &vframe_info,

23. sizeof(vframe_info));

24. }

编码线程 video_thread 取数据 ，编码 ，数据放到缓冲区

 

1. static void *video_thread(void *param)

2. {

3. struct video_output *video = param;

4.  

5. os_set_thread_name("video-io: video thread");

6.  

7. const char *video_thread_name =

8. profile_store_name(obs_get_profiler_name_store(),

9. "video_thread(%s)", video->info.name);

10.  

11. while (os_sem_wait(video->update_semaphore) == 0) {

12. if (video->stop)

13. break;

14.  

15. profile_start(video_thread_name);

16. while (!video->stop && !video_output_cur_frame(video)) {

17. video->total_frames++;

18. }

19.  

20. video->total_frames++;

21. profile_end(video_thread_name);

22.  

23. profile_reenable_thread();

24. }

25.  

26. return NULL;

27. }

 

1. static inline bool video_output_cur_frame(struct video_output *video)

2. {

3. struct cached_frame_info *frame_info;

4. bool complete;

5.  

6. /* -------------------------------- */

7.  

8. pthread_mutex_lock(&video->data_mutex);

9.  

10. frame_info = &video->cache[video->first_added];

11.  

12. pthread_mutex_unlock(&video->data_mutex);

13.  

14. /* -------------------------------- */

15.  

16. pthread_mutex_lock(&video->input_mutex);

17.  

18. for (size_t i = 0; i < video->inputs.num; i++) {

19. struct video_input *input = video->inputs.array+i;

20. struct video_data frame = frame_info->frame;

21.  

22. if (scale_video_output(input, &frame))

23. input->callback(input->param, &frame);

24. }

25.  

26. pthread_mutex_unlock(&video->input_mutex);

27.  

28. /* -------------------------------- */

29.  

30. pthread_mutex_lock(&video->data_mutex);

31.  

32. frame_info->frame.timestamp += video->frame_time;

33. complete = --frame_info->count == 0;

34.  

35. if (complete) {

36. if (++video->first_added == video->info.cache_size)

37. video->first_added = 0;

38.  

39. if (++video->available_frames == video->info.cache_size)

40. video->last_added = video->first_added;

41. }

42.  

43. pthread_mutex_unlock(&video->data_mutex);

44.  

45. /* -------------------------------- */

46.  

47. return complete;

48. }

input->callback对应 receive_video

 

1. static void receive_video(void *param, struct video_data *frame)

2. {

3. profile_start(receive_video_name);

4.  

5. struct obs_encoder *encoder = param;

6. struct obs_encoder *pair = encoder->paired_encoder;

7. struct encoder_frame enc_frame;

8.  

9. if (!encoder->first_received && pair) {

10. if (!pair->first_received ||

11. pair->first_raw_ts > frame->timestamp) {

12. goto wait_for_audio;

13. }

14. }

15.  

16. memset(&enc_frame, 0, sizeof(struct encoder_frame));

17.  

18. for (size_t i = 0; i < MAX_AV_PLANES; i++) {

19. enc_frame.data[i] = frame->data[i];

20. enc_frame.linesize[i] = frame->linesize[i];

21. }

22.  

23. if (!encoder->start_ts)

24. encoder->start_ts = frame->timestamp;

25.  

26. enc_frame.frames = 1;

27. enc_frame.pts = encoder->cur_pts;

28.  

29. do_encode(encoder, &enc_frame);

30.  

31. encoder->cur_pts += encoder->timebase_num;

32.  

33. wait_for_audio:

34. profile_end(receive_video_name);

35. }

最后编码的数据放到DARRAY(struct encoder_packet)   interleaved_packets;

 

1. static inline void insert_interleaved_packet(struct obs_output *output,

2. struct encoder_packet *out)

3. {

4. size_t idx;

5. for (idx = 0; idx < output->interleaved_packets.num; idx++) {

6. struct encoder_packet *cur_packet;

7. cur_packet = output->interleaved_packets.array + idx;

8.  

9. if (out->dts_usec < cur_packet->dts_usec)

10. break;

11. }

12.  

13. da_insert(output->interleaved_packets, idx, out);

14. }

 最后 send_thread 去缓冲数据发送，这块过程还有点问题，以后再看看