#include "recorder.h" #include #include #include #include #include #include "util/log.h" #include "util/str.h" /** Downcast packet sinks to recorder */ #define DOWNCAST_VIDEO(SINK) \ container_of(SINK, struct sc_recorder, video_packet_sink) #define DOWNCAST_AUDIO(SINK) \ container_of(SINK, struct sc_recorder, audio_packet_sink) static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us static const AVOutputFormat * find_muxer(const char *name) { #ifdef SCRCPY_LAVF_HAS_NEW_MUXER_ITERATOR_API void *opaque = NULL; #endif const AVOutputFormat *oformat = NULL; do { #ifdef SCRCPY_LAVF_HAS_NEW_MUXER_ITERATOR_API oformat = av_muxer_iterate(&opaque); #else oformat = av_oformat_next(oformat); #endif // until null or containing the requested name } while (oformat && !sc_str_list_contains(oformat->name, ',', name)); return oformat; } static AVPacket * sc_recorder_packet_ref(const AVPacket *packet) { AVPacket *p = av_packet_alloc(); if (!p) { LOG_OOM(); return NULL; } if (av_packet_ref(p, packet)) { av_packet_free(&p); return NULL; } return p; } static void sc_recorder_queue_clear(struct sc_recorder_queue *queue) { while (!sc_vecdeque_is_empty(queue)) { AVPacket *p = sc_vecdeque_pop(queue); av_packet_free(&p); } } static const char * sc_recorder_get_format_name(enum sc_record_format format) { switch (format) { case SC_RECORD_FORMAT_MP4: case SC_RECORD_FORMAT_M4A: case SC_RECORD_FORMAT_AAC: return "mp4"; case SC_RECORD_FORMAT_MKV: case SC_RECORD_FORMAT_MKA: return "matroska"; case SC_RECORD_FORMAT_OPUS: return "opus"; case SC_RECORD_FORMAT_FLAC: return "flac"; case SC_RECORD_FORMAT_WAV: return "wav"; default: return NULL; } } static bool sc_recorder_set_extradata(AVStream *ostream, const AVPacket *packet) { uint8_t *extradata = av_malloc(packet->size * sizeof(uint8_t)); if (!extradata) { LOG_OOM(); return false; } // copy the first packet to the extra data memcpy(extradata, packet->data, packet->size); ostream->codecpar->extradata = extradata; ostream->codecpar->extradata_size = packet->size; return true; } static inline void sc_recorder_rescale_packet(AVStream *stream, AVPacket *packet) { av_packet_rescale_ts(packet, SCRCPY_TIME_BASE, stream->time_base); } static bool sc_recorder_write_stream(struct sc_recorder *recorder, struct sc_recorder_stream *st, AVPacket *packet) { AVStream *stream = recorder->ctx->streams[st->index]; sc_recorder_rescale_packet(stream, packet); if (st->last_pts != AV_NOPTS_VALUE && packet->pts <= st->last_pts) { LOGD("Fixing PTS non monotonically increasing in stream %d " "(%" PRIi64 " >= %" PRIi64 ")", st->index, st->last_pts, packet->pts); packet->pts = ++st->last_pts; packet->dts = packet->pts; } else { st->last_pts = packet->pts; } return av_interleaved_write_frame(recorder->ctx, packet) >= 0; } static inline bool sc_recorder_write_video(struct sc_recorder *recorder, AVPacket *packet) { return sc_recorder_write_stream(recorder, &recorder->video_stream, packet); } static inline bool sc_recorder_write_audio(struct sc_recorder *recorder, AVPacket *packet) { return sc_recorder_write_stream(recorder, &recorder->audio_stream, packet); } static bool sc_recorder_open_output_file(struct sc_recorder *recorder) { const char *format_name = sc_recorder_get_format_name(recorder->format); assert(format_name); const AVOutputFormat *format = find_muxer(format_name); if (!format) { LOGE("Could not find muxer"); return false; } recorder->ctx = avformat_alloc_context(); if (!recorder->ctx) { LOG_OOM(); return false; } int ret = avio_open(&recorder->ctx->pb, recorder->filename, AVIO_FLAG_WRITE); if (ret < 0) { LOGE("Failed to open output file: %s", recorder->filename); avformat_free_context(recorder->ctx); return false; } // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate() // returns (on purpose) a pointer-to-const, but AVFormatContext.oformat // still expects a pointer-to-non-const (it has not be updated accordingly) // recorder->ctx->oformat = (AVOutputFormat *) format; av_dict_set(&recorder->ctx->metadata, "comment", "Recorded by scrcpy " SCRCPY_VERSION, 0); LOGI("Recording started to %s file: %s", format_name, recorder->filename); return true; } static void sc_recorder_close_output_file(struct sc_recorder *recorder) { avio_close(recorder->ctx->pb); avformat_free_context(recorder->ctx); } static inline bool sc_recorder_must_wait_for_config_packets(struct sc_recorder *recorder) { if (recorder->video && sc_vecdeque_is_empty(&recorder->video_queue)) { // The video queue is empty return true; } if (recorder->audio && recorder->audio_expects_config_packet && sc_vecdeque_is_empty(&recorder->audio_queue)) { // The audio queue is empty (when audio is enabled) return true; } // No queue is empty return false; } static bool sc_recorder_process_header(struct sc_recorder *recorder) { sc_mutex_lock(&recorder->mutex); while (!recorder->stopped && ((recorder->video && !recorder->video_init) || (recorder->audio && !recorder->audio_init) || sc_recorder_must_wait_for_config_packets(recorder))) { sc_cond_wait(&recorder->cond, &recorder->mutex); } if (recorder->video && sc_vecdeque_is_empty(&recorder->video_queue)) { assert(recorder->stopped); // If the recorder is stopped, don't process anything if there are not // at least video packets sc_mutex_unlock(&recorder->mutex); return false; } AVPacket *video_pkt = NULL; if (!sc_vecdeque_is_empty(&recorder->video_queue)) { assert(recorder->video); video_pkt = sc_vecdeque_pop(&recorder->video_queue); } AVPacket *audio_pkt = NULL; if (recorder->audio_expects_config_packet && !sc_vecdeque_is_empty(&recorder->audio_queue)) { assert(recorder->audio); audio_pkt = sc_vecdeque_pop(&recorder->audio_queue); } sc_mutex_unlock(&recorder->mutex); int ret = false; if (video_pkt) { if (video_pkt->pts != AV_NOPTS_VALUE) { LOGE("The first video packet is not a config packet"); goto end; } assert(recorder->video_stream.index >= 0); AVStream *video_stream = recorder->ctx->streams[recorder->video_stream.index]; bool ok = sc_recorder_set_extradata(video_stream, video_pkt); if (!ok) { goto end; } } if (audio_pkt) { if (audio_pkt->pts != AV_NOPTS_VALUE) { LOGE("The first audio packet is not a config packet"); goto end; } assert(recorder->audio_stream.index >= 0); AVStream *audio_stream = recorder->ctx->streams[recorder->audio_stream.index]; bool ok = sc_recorder_set_extradata(audio_stream, audio_pkt); if (!ok) { goto end; } } bool ok = avformat_write_header(recorder->ctx, NULL) >= 0; if (!ok) { LOGE("Failed to write header to %s", recorder->filename); goto end; } ret = true; end: if (video_pkt) { av_packet_free(&video_pkt); } if (audio_pkt) { av_packet_free(&audio_pkt); } return ret; } static bool sc_recorder_process_packets(struct sc_recorder *recorder) { int64_t pts_origin = AV_NOPTS_VALUE; bool header_written = sc_recorder_process_header(recorder); if (!header_written) { return false; } AVPacket *video_pkt = NULL; AVPacket *audio_pkt = NULL; // We can write a video packet only once we received the next one so that // we can set its duration (next_pts - current_pts) AVPacket *video_pkt_previous = NULL; bool error = false; for (;;) { sc_mutex_lock(&recorder->mutex); while (!recorder->stopped) { if (recorder->video && !video_pkt && !sc_vecdeque_is_empty(&recorder->video_queue)) { // A new packet may be assigned to video_pkt and be processed break; } if (recorder->audio && !audio_pkt && !sc_vecdeque_is_empty(&recorder->audio_queue)) { // A new packet may be assigned to audio_pkt and be processed break; } sc_cond_wait(&recorder->cond, &recorder->mutex); } // If stopped is set, continue to process the remaining events (to // finish the recording) before actually stopping. // If there is no video, then the video_queue will remain empty forever // and video_pkt will always be NULL. assert(recorder->video || (!video_pkt && sc_vecdeque_is_empty(&recorder->video_queue))); // If there is no audio, then the audio_queue will remain empty forever // and audio_pkt will always be NULL. assert(recorder->audio || (!audio_pkt && sc_vecdeque_is_empty(&recorder->audio_queue))); if (!video_pkt && !sc_vecdeque_is_empty(&recorder->video_queue)) { video_pkt = sc_vecdeque_pop(&recorder->video_queue); } if (!audio_pkt && !sc_vecdeque_is_empty(&recorder->audio_queue)) { audio_pkt = sc_vecdeque_pop(&recorder->audio_queue); } if (recorder->stopped && !video_pkt && !audio_pkt) { assert(sc_vecdeque_is_empty(&recorder->video_queue)); assert(sc_vecdeque_is_empty(&recorder->audio_queue)); sc_mutex_unlock(&recorder->mutex); break; } assert(video_pkt || audio_pkt); // at least one sc_mutex_unlock(&recorder->mutex); // Ignore further config packets (e.g. on device orientation // change). The next non-config packet will have the config packet // data prepended. if (video_pkt && video_pkt->pts == AV_NOPTS_VALUE) { av_packet_free(&video_pkt); video_pkt = NULL; } if (audio_pkt && audio_pkt->pts == AV_NOPTS_VALUE) { av_packet_free(&audio_pkt); audio_pkt = NULL; } if (pts_origin == AV_NOPTS_VALUE) { if (!recorder->audio) { assert(video_pkt); pts_origin = video_pkt->pts; } else if (!recorder->video) { assert(audio_pkt); pts_origin = audio_pkt->pts; } else if (video_pkt && audio_pkt) { pts_origin = MIN(video_pkt->pts, audio_pkt->pts); } else if (recorder->stopped) { if (video_pkt) { // The recorder is stopped without audio, record the video // packets pts_origin = video_pkt->pts; } else { // Fail if there is no video error = true; goto end; } } else { // We need both video and audio packets to initialize pts_origin continue; } } assert(pts_origin != AV_NOPTS_VALUE); if (video_pkt) { video_pkt->pts -= pts_origin; video_pkt->dts = video_pkt->pts; if (video_pkt_previous) { // we now know the duration of the previous packet video_pkt_previous->duration = video_pkt->pts - video_pkt_previous->pts; bool ok = sc_recorder_write_video(recorder, video_pkt_previous); av_packet_free(&video_pkt_previous); if (!ok) { LOGE("Could not record video packet"); error = true; goto end; } } video_pkt_previous = video_pkt; video_pkt = NULL; } if (audio_pkt) { audio_pkt->pts -= pts_origin; audio_pkt->dts = audio_pkt->pts; bool ok = sc_recorder_write_audio(recorder, audio_pkt); if (!ok) { LOGE("Could not record audio packet"); error = true; goto end; } av_packet_free(&audio_pkt); audio_pkt = NULL; } } // Write the last video packet AVPacket *last = video_pkt_previous; if (last) { // assign an arbitrary duration to the last packet last->duration = 100000; bool ok = sc_recorder_write_video(recorder, last); if (!ok) { // failing to write the last frame is not very serious, no // future frame may depend on it, so the resulting file // will still be valid LOGW("Could not record last packet"); } av_packet_free(&last); } int ret = av_write_trailer(recorder->ctx); if (ret < 0) { LOGE("Failed to write trailer to %s", recorder->filename); error = false; } end: if (video_pkt) { av_packet_free(&video_pkt); } if (audio_pkt) { av_packet_free(&audio_pkt); } return !error; } static bool sc_recorder_record(struct sc_recorder *recorder) { bool ok = sc_recorder_open_output_file(recorder); if (!ok) { return false; } ok = sc_recorder_process_packets(recorder); sc_recorder_close_output_file(recorder); return ok; } static int run_recorder(void *data) { struct sc_recorder *recorder = data; // Recording is a background task bool ok = sc_thread_set_priority(SC_THREAD_PRIORITY_LOW); (void) ok; // We don't care if it worked bool success = sc_recorder_record(recorder); sc_mutex_lock(&recorder->mutex); // Prevent the producer to push any new packet recorder->stopped = true; // Discard pending packets sc_recorder_queue_clear(&recorder->video_queue); sc_recorder_queue_clear(&recorder->audio_queue); sc_mutex_unlock(&recorder->mutex); if (success) { const char *format_name = sc_recorder_get_format_name(recorder->format); LOGI("Recording complete to %s file: %s", format_name, recorder->filename); } else { LOGE("Recording failed to %s", recorder->filename); } LOGD("Recorder thread ended"); recorder->cbs->on_ended(recorder, success, recorder->cbs_userdata); return 0; } static bool sc_recorder_set_orientation(AVStream *stream, enum sc_orientation orientation) { assert(!sc_orientation_is_mirror(orientation)); uint8_t *raw_data; #ifdef SCRCPY_LAVC_HAS_CODECPAR_CODEC_SIDEDATA AVPacketSideData *sd = av_packet_side_data_new(&stream->codecpar->coded_side_data, &stream->codecpar->nb_coded_side_data, AV_PKT_DATA_DISPLAYMATRIX, sizeof(int32_t) * 9, 0); if (!sd) { LOG_OOM(); return false; } raw_data = sd->data; #else raw_data = av_stream_new_side_data(stream, AV_PKT_DATA_DISPLAYMATRIX, sizeof(int32_t) * 9); if (!raw_data) { LOG_OOM(); return false; } #endif int32_t *matrix = (int32_t *) raw_data; unsigned rotation = orientation; unsigned angle = rotation * 90; av_display_rotation_set(matrix, angle); return true; } static bool sc_recorder_video_packet_sink_open(struct sc_packet_sink *sink, AVCodecContext *ctx) { struct sc_recorder *recorder = DOWNCAST_VIDEO(sink); // only written from this thread, no need to lock assert(!recorder->video_init); sc_mutex_lock(&recorder->mutex); if (recorder->stopped) { sc_mutex_unlock(&recorder->mutex); return false; } AVStream *stream = avformat_new_stream(recorder->ctx, ctx->codec); if (!stream) { sc_mutex_unlock(&recorder->mutex); return false; } int r = avcodec_parameters_from_context(stream->codecpar, ctx); if (r < 0) { sc_mutex_unlock(&recorder->mutex); return false; } recorder->video_stream.index = stream->index; if (recorder->orientation != SC_ORIENTATION_0) { if (!sc_recorder_set_orientation(stream, recorder->orientation)) { sc_mutex_unlock(&recorder->mutex); return false; } LOGI("Record orientation set to %s", sc_orientation_get_name(recorder->orientation)); } recorder->video_init = true; sc_cond_signal(&recorder->cond); sc_mutex_unlock(&recorder->mutex); return true; } static void sc_recorder_video_packet_sink_close(struct sc_packet_sink *sink) { struct sc_recorder *recorder = DOWNCAST_VIDEO(sink); // only written from this thread, no need to lock assert(recorder->video_init); sc_mutex_lock(&recorder->mutex); // EOS also stops the recorder recorder->stopped = true; sc_cond_signal(&recorder->cond); sc_mutex_unlock(&recorder->mutex); } static bool sc_recorder_video_packet_sink_push(struct sc_packet_sink *sink, const AVPacket *packet) { struct sc_recorder *recorder = DOWNCAST_VIDEO(sink); // only written from this thread, no need to lock assert(recorder->video_init); sc_mutex_lock(&recorder->mutex); if (recorder->stopped) { // reject any new packet sc_mutex_unlock(&recorder->mutex); return false; } AVPacket *rec = sc_recorder_packet_ref(packet); if (!rec) { LOG_OOM(); sc_mutex_unlock(&recorder->mutex); return false; } rec->stream_index = recorder->video_stream.index; bool ok = sc_vecdeque_push(&recorder->video_queue, rec); if (!ok) { LOG_OOM(); sc_mutex_unlock(&recorder->mutex); return false; } sc_cond_signal(&recorder->cond); sc_mutex_unlock(&recorder->mutex); return true; } static bool sc_recorder_audio_packet_sink_open(struct sc_packet_sink *sink, AVCodecContext *ctx) { struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); assert(recorder->audio); // only written from this thread, no need to lock assert(!recorder->audio_init); sc_mutex_lock(&recorder->mutex); AVStream *stream = avformat_new_stream(recorder->ctx, ctx->codec); if (!stream) { sc_mutex_unlock(&recorder->mutex); return false; } int r = avcodec_parameters_from_context(stream->codecpar, ctx); if (r < 0) { sc_mutex_unlock(&recorder->mutex); return false; } recorder->audio_stream.index = stream->index; // A config packet is provided for all supported formats except raw audio recorder->audio_expects_config_packet = ctx->codec_id != AV_CODEC_ID_PCM_S16LE; recorder->audio_init = true; sc_cond_signal(&recorder->cond); sc_mutex_unlock(&recorder->mutex); return true; } static void sc_recorder_audio_packet_sink_close(struct sc_packet_sink *sink) { struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); assert(recorder->audio); // only written from this thread, no need to lock assert(recorder->audio_init); sc_mutex_lock(&recorder->mutex); // EOS also stops the recorder recorder->stopped = true; sc_cond_signal(&recorder->cond); sc_mutex_unlock(&recorder->mutex); } static bool sc_recorder_audio_packet_sink_push(struct sc_packet_sink *sink, const AVPacket *packet) { struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); assert(recorder->audio); // only written from this thread, no need to lock assert(recorder->audio_init); sc_mutex_lock(&recorder->mutex); if (recorder->stopped) { // reject any new packet sc_mutex_unlock(&recorder->mutex); return false; } AVPacket *rec = sc_recorder_packet_ref(packet); if (!rec) { LOG_OOM(); sc_mutex_unlock(&recorder->mutex); return false; } rec->stream_index = recorder->audio_stream.index; bool ok = sc_vecdeque_push(&recorder->audio_queue, rec); if (!ok) { LOG_OOM(); sc_mutex_unlock(&recorder->mutex); return false; } sc_cond_signal(&recorder->cond); sc_mutex_unlock(&recorder->mutex); return true; } static void sc_recorder_audio_packet_sink_disable(struct sc_packet_sink *sink) { struct sc_recorder *recorder = DOWNCAST_AUDIO(sink); assert(recorder->audio); // only written from this thread, no need to lock assert(!recorder->audio_init); LOGW("Audio stream recording disabled"); sc_mutex_lock(&recorder->mutex); recorder->audio = false; recorder->audio_init = true; sc_cond_signal(&recorder->cond); sc_mutex_unlock(&recorder->mutex); } static void sc_recorder_stream_init(struct sc_recorder_stream *stream) { stream->index = -1; stream->last_pts = AV_NOPTS_VALUE; } bool sc_recorder_init(struct sc_recorder *recorder, const char *filename, enum sc_record_format format, bool video, bool audio, enum sc_orientation orientation, const struct sc_recorder_callbacks *cbs, void *cbs_userdata) { assert(!sc_orientation_is_mirror(orientation)); recorder->filename = strdup(filename); if (!recorder->filename) { LOG_OOM(); return false; } bool ok = sc_mutex_init(&recorder->mutex); if (!ok) { goto error_free_filename; } ok = sc_cond_init(&recorder->cond); if (!ok) { goto error_mutex_destroy; } assert(video || audio); recorder->video = video; recorder->audio = audio; recorder->orientation = orientation; sc_vecdeque_init(&recorder->video_queue); sc_vecdeque_init(&recorder->audio_queue); recorder->stopped = false; recorder->video_init = false; recorder->audio_init = false; recorder->audio_expects_config_packet = false; sc_recorder_stream_init(&recorder->video_stream); sc_recorder_stream_init(&recorder->audio_stream); recorder->format = format; assert(cbs && cbs->on_ended); recorder->cbs = cbs; recorder->cbs_userdata = cbs_userdata; if (video) { static const struct sc_packet_sink_ops video_ops = { .open = sc_recorder_video_packet_sink_open, .close = sc_recorder_video_packet_sink_close, .push = sc_recorder_video_packet_sink_push, }; recorder->video_packet_sink.ops = &video_ops; } if (audio) { static const struct sc_packet_sink_ops audio_ops = { .open = sc_recorder_audio_packet_sink_open, .close = sc_recorder_audio_packet_sink_close, .push = sc_recorder_audio_packet_sink_push, .disable = sc_recorder_audio_packet_sink_disable, }; recorder->audio_packet_sink.ops = &audio_ops; } return true; error_mutex_destroy: sc_mutex_destroy(&recorder->mutex); error_free_filename: free(recorder->filename); return false; } bool sc_recorder_start(struct sc_recorder *recorder) { bool ok = sc_thread_create(&recorder->thread, run_recorder, "scrcpy-recorder", recorder); if (!ok) { LOGE("Could not start recorder thread"); return false; } return true; } void sc_recorder_stop(struct sc_recorder *recorder) { sc_mutex_lock(&recorder->mutex); recorder->stopped = true; sc_cond_signal(&recorder->cond); sc_mutex_unlock(&recorder->mutex); } void sc_recorder_join(struct sc_recorder *recorder) { sc_thread_join(&recorder->thread, NULL); } void sc_recorder_destroy(struct sc_recorder *recorder) { sc_cond_destroy(&recorder->cond); sc_mutex_destroy(&recorder->mutex); free(recorder->filename); }