Pass video size as codec metadata

On initial connection, scrcpy sent some device metadata:
 - the device name (to be used as window title)
 - the initial video size (before any frame or even SPS/PPS)

But it is better to provide the initial video size as part as the video
stream, so that it can be demuxed and exposed via AVCodecContext to
sinks.

This avoids to pass an explicit "initial frame size" for the screen, the
recorder and the v4l2 sink.

app/src/demuxer.c

@@ -57,6 +57,20 @@ sc_demuxer_recv_codec_id(struct sc_demuxer *demuxer, uint32_t *codec_id) {
     return true;
 }
 
+static bool
+sc_demuxer_recv_video_size(struct sc_demuxer *demuxer, uint32_t *width,
+                           uint32_t *height) {
+    uint8_t data[8];
+    ssize_t r = net_recv_all(demuxer->socket, data, 8);
+    if (r < 8) {
+        return false;
+    }
+
+    *width = sc_read32be(data);
+    *height = sc_read32be(data + 4);
+    return true;
+}
+
 static bool
 sc_demuxer_recv_packet(struct sc_demuxer *demuxer, AVPacket *packet) {
     // The video stream contains raw packets, without time information. When we
@@ -169,7 +183,15 @@ run_demuxer(void *data) {
     codec_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
 
     if (codec->type == AVMEDIA_TYPE_VIDEO) {
-        // Hardcoded video properties
+        uint32_t width;
+        uint32_t height;
+        ok = sc_demuxer_recv_video_size(demuxer, &width, &height);
+        if (!ok) {
+            goto finally_free_context;
+        }
+
+        codec_ctx->width = width;
+        codec_ctx->height = height;
         codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
     } else {
         // Hardcoded audio properties

app/src/events.h

@@ -5,3 +5,4 @@
 #define SC_EVENT_USB_DEVICE_DISCONNECTED  (SDL_USEREVENT + 4)
 #define SC_EVENT_DEMUXER_ERROR            (SDL_USEREVENT + 5)
 #define SC_EVENT_RECORDER_ERROR           (SDL_USEREVENT + 6)
+#define SC_EVENT_SCREEN_INIT_SIZE         (SDL_USEREVENT + 7)

app/src/recorder.c

@@ -479,9 +479,6 @@ sc_recorder_video_packet_sink_open(struct sc_packet_sink *sink,
         return false;
     }
 
-    stream->codecpar->width = recorder->declared_frame_size.width;
-    stream->codecpar->height = recorder->declared_frame_size.height;
-
     recorder->video_stream_index = stream->index;
 
     recorder->video_init = true;
@@ -643,7 +640,6 @@ sc_recorder_audio_packet_sink_disable(struct sc_packet_sink *sink) {
 bool
 sc_recorder_init(struct sc_recorder *recorder, const char *filename,
                  enum sc_record_format format, bool audio,
-                 struct sc_size declared_frame_size,
                  const struct sc_recorder_callbacks *cbs, void *cbs_userdata) {
     recorder->filename = strdup(filename);
     if (!recorder->filename) {
@@ -679,7 +675,6 @@ sc_recorder_init(struct sc_recorder *recorder, const char *filename,
     recorder->audio_stream_index = -1;
 
     recorder->format = format;
-    recorder->declared_frame_size = declared_frame_size;
 
     assert(cbs && cbs->on_ended);
     recorder->cbs = cbs;

app/src/recorder.h

@@ -31,7 +31,6 @@ struct sc_recorder {
     char *filename;
     enum sc_record_format format;
     AVFormatContext *ctx;
-    struct sc_size declared_frame_size;
 
     sc_thread thread;
     sc_mutex mutex;
@@ -61,7 +60,6 @@ struct sc_recorder_callbacks {
 bool
 sc_recorder_init(struct sc_recorder *recorder, const char *filename,
                  enum sc_record_format format, bool audio,
-                 struct sc_size declared_frame_size,
                  const struct sc_recorder_callbacks *cbs, void *cbs_userdata);
 
 bool

app/src/scrcpy.c

@@ -473,7 +473,7 @@ scrcpy(struct scrcpy_options *options) {
         };
         if (!sc_recorder_init(&s->recorder, options->record_filename,
                               options->record_format, options->audio,
-                              info->frame_size, &recorder_cbs, NULL)) {
+                              &recorder_cbs, NULL)) {
             goto end;
         }
         recorder_initialized = true;
@@ -660,7 +660,6 @@ aoa_hid_end:
             .clipboard_autosync = options->clipboard_autosync,
             .shortcut_mods = &options->shortcut_mods,
             .window_title = window_title,
-            .frame_size = info->frame_size,
             .always_on_top = options->always_on_top,
             .window_x = options->window_x,
             .window_y = options->window_y,
@@ -697,8 +696,7 @@ aoa_hid_end:
 
 #ifdef HAVE_V4L2
     if (options->v4l2_device) {
-        if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device,
-                               info->frame_size)) {
+        if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device)) {
             goto end;
         }
 

app/src/screen.c

@@ -239,7 +239,7 @@ sc_screen_update_content_rect(struct sc_screen *screen) {
     }
 }
 
-static inline SDL_Texture *
+static bool
 create_texture(struct sc_screen *screen) {
     SDL_Renderer *renderer = screen->renderer;
     struct sc_size size = screen->frame_size;
@@ -247,7 +247,8 @@ create_texture(struct sc_screen *screen) {
                                              SDL_TEXTUREACCESS_STREAMING,
                                              size.width, size.height);
     if (!texture) {
-        return NULL;
+        LOGE("Could not create texture: %s", SDL_GetError());
+        return false;
     }
 
     if (screen->mipmaps) {
@@ -263,7 +264,8 @@ create_texture(struct sc_screen *screen) {
         SDL_GL_UnbindTexture(texture);
     }
 
-    return texture;
+    screen->texture = texture;
+    return true;
 }
 
 // render the texture to the renderer
@@ -335,7 +337,25 @@ sc_screen_frame_sink_open(struct sc_frame_sink *sink,
     (void) ctx;
 
     struct sc_screen *screen = DOWNCAST(sink);
-    (void) screen;
+
+    assert(ctx->width > 0 && ctx->width <= 0xFFFF);
+    assert(ctx->height > 0 && ctx->height <= 0xFFFF);
+    // screen->frame_size is never used before the event is pushed, and the
+    // event acts as a memory barrier so it is safe without mutex
+    screen->frame_size.width = ctx->width;
+    screen->frame_size.height = ctx->height;
+
+    static SDL_Event event = {
+        .type = SC_EVENT_SCREEN_INIT_SIZE,
+    };
+
+    // Post the event on the UI thread (the texture must be created from there)
+    int ret = SDL_PushEvent(&event);
+    if (ret < 0) {
+        LOGW("Could not post init size event: %s", SDL_GetError());
+        return false;
+    }
+
 #ifndef NDEBUG
     screen->open = true;
 #endif
@@ -410,14 +430,10 @@ sc_screen_init(struct sc_screen *screen,
         goto error_destroy_frame_buffer;
     }
 
-    screen->frame_size = params->frame_size;
     screen->rotation = params->rotation;
     if (screen->rotation) {
         LOGI("Initial display rotation set to %u", screen->rotation);
     }
-    struct sc_size content_size =
-        get_rotated_size(screen->frame_size, screen->rotation);
-    screen->content_size = content_size;
 
     uint32_t window_flags = SDL_WINDOW_HIDDEN
                           | SDL_WINDOW_RESIZABLE
@@ -485,18 +501,10 @@ sc_screen_init(struct sc_screen *screen,
         LOGW("Could not load icon");
     }
 
-    LOGI("Initial texture: %" PRIu16 "x%" PRIu16, params->frame_size.width,
-                                                  params->frame_size.height);
-    screen->texture = create_texture(screen);
-    if (!screen->texture) {
-        LOGE("Could not create texture: %s", SDL_GetError());
-        goto error_destroy_renderer;
-    }
-
     screen->frame = av_frame_alloc();
     if (!screen->frame) {
         LOG_OOM();
-        goto error_destroy_texture;
+        goto error_destroy_renderer;
     }
 
     struct sc_input_manager_params im_params = {
@@ -531,8 +539,6 @@ sc_screen_init(struct sc_screen *screen,
 
     return true;
 
-error_destroy_texture:
-    SDL_DestroyTexture(screen->texture);
 error_destroy_renderer:
     SDL_DestroyRenderer(screen->renderer);
 error_destroy_window:
@@ -591,7 +597,9 @@ sc_screen_destroy(struct sc_screen *screen) {
     assert(!screen->open);
 #endif
     av_frame_free(&screen->frame);
-    SDL_DestroyTexture(screen->texture);
+    if (screen->texture) {
+        SDL_DestroyTexture(screen->texture);
+    }
     SDL_DestroyRenderer(screen->renderer);
     SDL_DestroyWindow(screen->window);
     sc_fps_counter_destroy(&screen->fps_counter);
@@ -655,6 +663,23 @@ sc_screen_set_rotation(struct sc_screen *screen, unsigned rotation) {
     sc_screen_render(screen, true);
 }
 
+static bool
+sc_screen_init_size(struct sc_screen *screen) {
+    // Before first frame
+    assert(!screen->has_frame);
+    assert(!screen->texture);
+
+    // The requested size is passed via screen->frame_size
+
+    struct sc_size content_size =
+        get_rotated_size(screen->frame_size, screen->rotation);
+    screen->content_size = content_size;
+
+    LOGI("Initial texture: %" PRIu16 "x%" PRIu16,
+                 screen->frame_size.width, screen->frame_size.height);
+    return create_texture(screen);
+}
+
 // recreate the texture and resize the window if the frame size has changed
 static bool
 prepare_for_frame(struct sc_screen *screen, struct sc_size new_frame_size) {
@@ -673,11 +698,7 @@ prepare_for_frame(struct sc_screen *screen, struct sc_size new_frame_size) {
 
         LOGI("New texture: %" PRIu16 "x%" PRIu16,
                      screen->frame_size.width, screen->frame_size.height);
-        screen->texture = create_texture(screen);
-        if (!screen->texture) {
-            LOGE("Could not create texture: %s", SDL_GetError());
-            return false;
-        }
+        return create_texture(screen);
     }
 
     return true;
@@ -795,6 +816,14 @@ sc_screen_handle_event(struct sc_screen *screen, SDL_Event *event) {
     bool relative_mode = sc_screen_is_relative_mode(screen);
 
     switch (event->type) {
+        case SC_EVENT_SCREEN_INIT_SIZE:
+            // The initial size is passed via screen->frame_size
+            bool ok = sc_screen_init_size(screen);
+            if (!ok) {
+                LOGE("Could not initialize screen size");
+                return false;
+            }
+            return true;
         case SC_EVENT_NEW_FRAME: {
             bool ok = sc_screen_update_frame(screen);
             if (!ok) {

app/src/screen.h

@@ -78,7 +78,6 @@ struct sc_screen_params {
     const struct sc_shortcut_mods *shortcut_mods;
 
     const char *window_title;
-    struct sc_size frame_size;
     bool always_on_top;
 
     int16_t window_x; // accepts SC_WINDOW_POSITION_UNDEFINED

app/src/server.c

@@ -441,9 +441,9 @@ sc_server_init(struct sc_server *server, const struct sc_server_params *params,
 static bool
 device_read_info(struct sc_intr *intr, sc_socket device_socket,
                  struct sc_server_info *info) {
-    unsigned char buf[SC_DEVICE_NAME_FIELD_LENGTH + 4];
+    unsigned char buf[SC_DEVICE_NAME_FIELD_LENGTH];
     ssize_t r = net_recv_all_intr(intr, device_socket, buf, sizeof(buf));
-    if (r < SC_DEVICE_NAME_FIELD_LENGTH + 4) {
+    if (r < SC_DEVICE_NAME_FIELD_LENGTH) {
         LOGE("Could not retrieve device information");
         return false;
     }
@@ -451,9 +451,6 @@ device_read_info(struct sc_intr *intr, sc_socket device_socket,
     buf[SC_DEVICE_NAME_FIELD_LENGTH - 1] = '\0';
     memcpy(info->device_name, (char *) buf, sizeof(info->device_name));
 
-    unsigned char *fields = &buf[SC_DEVICE_NAME_FIELD_LENGTH];
-    info->frame_size.width = sc_read16be(fields);
-    info->frame_size.height = sc_read16be(&fields[2]);
     return true;
 }
 

app/src/server.h

@@ -18,7 +18,6 @@
 #define SC_DEVICE_NAME_FIELD_LENGTH 64
 struct sc_server_info {
     char device_name[SC_DEVICE_NAME_FIELD_LENGTH];
-    struct sc_size frame_size;
 };
 
 struct sc_server_params {

app/src/v4l2_sink.c

@@ -210,9 +210,6 @@ sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) {
         goto error_avformat_free_context;
     }
 
-    ostream->codecpar->width = vs->frame_size.width;
-    ostream->codecpar->height = vs->frame_size.height;
-
     int ret = avio_open(&vs->format_ctx->pb, vs->device_name, AVIO_FLAG_WRITE);
     if (ret < 0) {
         LOGE("Failed to open output device: %s", vs->device_name);
@@ -226,8 +223,8 @@ sc_v4l2_sink_open(struct sc_v4l2_sink *vs, const AVCodecContext *ctx) {
         goto error_avio_close;
     }
 
-    vs->encoder_ctx->width = vs->frame_size.width;
-    vs->encoder_ctx->height = vs->frame_size.height;
+    vs->encoder_ctx->width = ctx->width;
+    vs->encoder_ctx->height = ctx->height;
     vs->encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
     vs->encoder_ctx->time_base.num = 1;
     vs->encoder_ctx->time_base.den = 1;
@@ -343,16 +340,13 @@ sc_v4l2_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) {
 }
 
 bool
-sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
-                  struct sc_size frame_size) {
+sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name) {
     vs->device_name = strdup(device_name);
     if (!vs->device_name) {
         LOGE("Could not strdup v4l2 device name");
         return false;
     }
 
-    vs->frame_size = frame_size;
-
     static const struct sc_frame_sink_ops ops = {
         .open = sc_v4l2_frame_sink_open,
         .close = sc_v4l2_frame_sink_close,

app/src/v4l2_sink.h

@@ -19,7 +19,6 @@ struct sc_v4l2_sink {
     AVCodecContext *encoder_ctx;
 
     char *device_name;
-    struct sc_size frame_size;
 
     sc_thread thread;
     sc_mutex mutex;
@@ -33,8 +32,7 @@ struct sc_v4l2_sink {
 };
 
 bool
-sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name,
-                  struct sc_size frame_size);
+sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name);
 
 void
 sc_v4l2_sink_destroy(struct sc_v4l2_sink *vs);

server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java

@@ -101,7 +101,7 @@ public final class AudioEncoder implements AsyncProcessor {
     }
 
     private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException {
-        streamer.writeHeader();
+        streamer.writeAudioHeader();
 
         while (!Thread.currentThread().isInterrupted()) {
             OutputTask task = outputTasks.take();

server/src/main/java/com/genymobile/scrcpy/AudioRawRecorder.java

@@ -26,7 +26,7 @@ public final class AudioRawRecorder implements AsyncProcessor {
         try {
             capture.start();
 
-            streamer.writeHeader();
+            streamer.writeAudioHeader();
             while (!Thread.currentThread().isInterrupted()) {
                 buffer.position(0);
                 int r = capture.read(buffer, READ_SIZE, bufferInfo);

server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java

@@ -122,18 +122,14 @@ public final class DesktopConnection implements Closeable {
         }
     }
 
-    public void sendDeviceMeta(String deviceName, int width, int height) throws IOException {
-        byte[] buffer = new byte[DEVICE_NAME_FIELD_LENGTH + 4];
+    public void sendDeviceMeta(String deviceName) throws IOException {
+        byte[] buffer = new byte[DEVICE_NAME_FIELD_LENGTH];
 
         byte[] deviceNameBytes = deviceName.getBytes(StandardCharsets.UTF_8);
         int len = StringUtils.getUtf8TruncationIndex(deviceNameBytes, DEVICE_NAME_FIELD_LENGTH - 1);
         System.arraycopy(deviceNameBytes, 0, buffer, 0, len);
         // byte[] are always 0-initialized in java, no need to set '\0' explicitly
 
-        buffer[DEVICE_NAME_FIELD_LENGTH] = (byte) (width >> 8);
-        buffer[DEVICE_NAME_FIELD_LENGTH + 1] = (byte) width;
-        buffer[DEVICE_NAME_FIELD_LENGTH + 2] = (byte) (height >> 8);
-        buffer[DEVICE_NAME_FIELD_LENGTH + 3] = (byte) height;
         IO.writeFully(videoFd, buffer, 0, buffer.length);
     }
 

server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java

@@ -66,7 +66,7 @@ public class ScreenEncoder implements Device.RotationListener {
         IBinder display = createDisplay();
         device.setRotationListener(this);
 
-        streamer.writeHeader();
+        streamer.writeVideoHeader(device.getScreenInfo().getVideoSize());
 
         boolean alive;
         try {

server/src/main/java/com/genymobile/scrcpy/Server.java

@@ -96,8 +96,7 @@ public final class Server {
 
         try (DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, audio, control, sendDummyByte)) {
             if (options.getSendDeviceMeta()) {
-                Size videoSize = device.getScreenInfo().getVideoSize();
-                connection.sendDeviceMeta(Device.getDeviceName(), videoSize.getWidth(), videoSize.getHeight());
+                connection.sendDeviceMeta(Device.getDeviceName());
             }
 
             if (control) {

server/src/main/java/com/genymobile/scrcpy/Streamer.java

@@ -30,8 +30,7 @@ public final class Streamer {
     public Codec getCodec() {
         return codec;
     }
-
-    public void writeHeader() throws IOException {
+    public void writeAudioHeader() throws IOException {
         if (sendCodecMeta) {
             ByteBuffer buffer = ByteBuffer.allocate(4);
             buffer.putInt(codec.getId());
@@ -40,6 +39,17 @@ public final class Streamer {
         }
     }
 
+    public void writeVideoHeader(Size videoSize) throws IOException {
+        if (sendCodecMeta) {
+            ByteBuffer buffer = ByteBuffer.allocate(12);
+            buffer.putInt(codec.getId());
+            buffer.putInt(videoSize.getWidth());
+            buffer.putInt(videoSize.getHeight());
+            buffer.flip();
+            IO.writeFully(fd, buffer);
+        }
+    }
+
     public void writeDisableStream(boolean error) throws IOException {
         // Writing a specific code as codec-id means that the device disables the stream
         //   code 0: it explicitly disables the stream (because it could not capture audio), scrcpy should continue mirroring video only