#include "scrcpy.h" #include #include #include #include #include #include #include #include "command.h" #include "common.h" #include "control.h" #include "convert.h" #include "decoder.h" #include "events.h" #include "frames.h" #include "lockutil.h" #include "netutil.h" #include "server.h" #include "tinyxpm.h" #include "icon.xpm" #define DEVICE_NAME_FIELD_LENGTH 64 #define DISPLAY_MARGINS 96 static struct frames frames; static struct decoder decoder; static struct controller controller; static SDL_Window *window; static SDL_Renderer *renderer; static SDL_Texture *texture; static struct size frame_size; // used only in fullscreen mode to know the windowed window size static struct size windowed_window_size; static SDL_bool texture_empty = SDL_TRUE; static SDL_bool fullscreen = SDL_FALSE; static long timestamp_ms(void) { struct timeval tv; gettimeofday(&tv, NULL); return tv.tv_sec * 1000 + tv.tv_usec / 1000; } static void count_frame(void) { static long ts = 0; static int nbframes = 0; long now = timestamp_ms(); ++nbframes; if (now - ts > 1000) { SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "%d fps", nbframes); ts = now; nbframes = 0; } } static TCPsocket listen_on_port(Uint16 port) { IPaddress addr = { .host = INADDR_ANY, .port = SDL_SwapBE16(port), }; return SDLNet_TCP_Open(&addr); } // name must be at least DEVICE_NAME_FIELD_LENGTH bytes static SDL_bool read_initial_device_info(TCPsocket socket, char *device_name, struct size *size) { unsigned char buf[DEVICE_NAME_FIELD_LENGTH + 4]; if (SDLNet_TCP_Recv(socket, buf, sizeof(buf)) <= 0) { return SDL_FALSE; } buf[DEVICE_NAME_FIELD_LENGTH - 1] = '\0'; // in case the client sends garbage // scrcpy is safe here, since name contains at least DEVICE_NAME_FIELD_LENGTH bytes // and strlen(buf) < DEVICE_NAME_FIELD_LENGTH strcpy(device_name, (char *) buf); size->width = (buf[DEVICE_NAME_FIELD_LENGTH] << 8) | buf[DEVICE_NAME_FIELD_LENGTH + 1]; size->height = (buf[DEVICE_NAME_FIELD_LENGTH + 2] << 8) | buf[DEVICE_NAME_FIELD_LENGTH + 3]; return SDL_TRUE; } #if SDL_VERSION_ATLEAST(2, 0, 5) # define GET_DISPLAY_BOUNDS(i, r) SDL_GetDisplayUsableBounds((i), (r)) #else # define GET_DISPLAY_BOUNDS(i, r) SDL_GetDisplayBounds((i), (r)) #endif // init the preferred display_bounds (i.e. the screen bounds with some margins) static SDL_bool get_preferred_display_bounds(struct size *bounds) { SDL_Rect rect; if (GET_DISPLAY_BOUNDS(0, &rect)) { SDL_LogWarn(SDL_LOG_CATEGORY_SYSTEM, "Could not get display usable bounds: %s", SDL_GetError()); return SDL_FALSE; } bounds->width = MAX(0, rect.w - DISPLAY_MARGINS); bounds->height = MAX(0, rect.h - DISPLAY_MARGINS); return SDL_TRUE; } static inline struct size get_window_size(SDL_Window *window) { int width; int height; SDL_GetWindowSize(window, &width, &height); struct size size; size.width = width; size.height = height; return size; } static void set_window_size(SDL_Window *window, struct size new_size) { // setting the window size during fullscreen is implementation defined, // so apply the resize only after fullscreen is disabled if (fullscreen) { // SDL_SetWindowSize will be called when fullscreen will be disabled windowed_window_size = new_size; } else { SDL_SetWindowSize(window, new_size.width, new_size.height); } } static inline struct point get_mouse_point() { int x; int y; SDL_GetMouseState(&x, &y); SDL_assert_release(x >= 0 && x < 0x10000 && y >= 0 && y < 0x10000); return (struct point) { .x = (Uint16) x, .y = (Uint16) y, }; } // return the optimal size of the window, with the following constraints: // - it attempts to keep at least one dimension of the current_size (i.e. it crops the black borders) // - it keeps the aspect ratio // - it scales down to make it fit in the display_size static struct size get_optimal_size(struct size current_size, struct size frame_size) { struct size display_size; // 32 bits because we need to multiply two 16 bits values Uint32 w; Uint32 h; if (!get_preferred_display_bounds(&display_size)) { // cannot get display bounds, do not constraint the size w = current_size.width; h = current_size.height; } else { w = MIN(current_size.width, display_size.width); h = MIN(current_size.height, display_size.height); } SDL_bool keep_width = frame_size.width * h > frame_size.height * w; if (keep_width) { // remove black borders on top and bottom h = frame_size.height * w / frame_size.width; } else { // remove black borders on left and right (or none at all if it already fits) w = frame_size.width * h / frame_size.height; } // w and h must fit into 16 bits SDL_assert_release(w < 0x10000 && h < 0x10000); return (struct size) {w, h}; } // initially, there is no current size, so use the frame size as current size static inline struct size get_initial_optimal_size(struct size frame_size) { return get_optimal_size(frame_size, frame_size); } // same as get_optimal_size(), but read the current size from the window static inline struct size get_optimal_window_size(SDL_Window *window, struct size frame_size) { struct size current_size = get_window_size(window); return get_optimal_size(current_size, frame_size); } static SDL_bool prepare_for_frame(SDL_Window *window, SDL_Renderer *renderer, SDL_Texture **texture, struct size old_frame_size, struct size frame_size) { if (old_frame_size.width != frame_size.width || old_frame_size.height != frame_size.height) { if (SDL_RenderSetLogicalSize(renderer, frame_size.width, frame_size.height)) { SDL_LogError(SDL_LOG_CATEGORY_RENDER, "Could not set renderer logical size: %s", SDL_GetError()); return SDL_FALSE; } // frame dimension changed, destroy texture SDL_DestroyTexture(*texture); struct size current_size = get_window_size(window); struct size target_size = { (Uint32) current_size.width * frame_size.width / old_frame_size.width, (Uint32) current_size.height * frame_size.height / old_frame_size.height, }; target_size = get_optimal_size(target_size, frame_size); set_window_size(window, target_size); SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "New texture: %" PRIu16 "x%" PRIu16, frame_size.width, frame_size.height); *texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_YV12, SDL_TEXTUREACCESS_STREAMING, frame_size.width, frame_size.height); if (!*texture) { SDL_LogCritical(SDL_LOG_CATEGORY_RENDER, "Could not create texture: %s", SDL_GetError()); return SDL_FALSE; } } return SDL_TRUE; } static void update_texture(const AVFrame *frame, SDL_Texture *texture) { SDL_UpdateYUVTexture(texture, NULL, frame->data[0], frame->linesize[0], frame->data[1], frame->linesize[1], frame->data[2], frame->linesize[2]); } static void render(SDL_Renderer *renderer, SDL_Texture *texture) { SDL_RenderClear(renderer); if (texture) { SDL_RenderCopy(renderer, texture, NULL, NULL); } SDL_RenderPresent(renderer); } static void switch_fullscreen(void) { if (!fullscreen) { // going to fullscreen, store the current windowed window size windowed_window_size = get_window_size(window); } Uint32 new_mode = fullscreen ? 0 : SDL_WINDOW_FULLSCREEN_DESKTOP; if (SDL_SetWindowFullscreen(window, new_mode)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Could not switch fullscreen mode: %s", SDL_GetError()); return; } fullscreen = !fullscreen; if (!fullscreen) { // fullscreen disabled, restore expected windowed window size SDL_SetWindowSize(window, windowed_window_size.width, windowed_window_size.height); } SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "Switched to %s mode", fullscreen ? "fullscreen" : "windowed"); render(renderer, texture_empty ? NULL : texture); } static void resize_to_fit(void) { if (!fullscreen) { struct size optimal_size = get_optimal_window_size(window, frame_size); SDL_SetWindowSize(window, optimal_size.width, optimal_size.height); SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "Resized to optimal size"); } } static void resize_to_pixel_perfect(void) { if (!fullscreen) { SDL_SetWindowSize(window, frame_size.width, frame_size.height); SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "Resized to pixel-perfect"); } } static int wait_for_success(process_t proc, const char *name) { if (proc == PROCESS_NONE) { SDL_LogError(SDL_LOG_CATEGORY_SYSTEM, "Could not execute \"%s\"", name); return -1; } exit_code_t exit_code; if (!cmd_simple_wait(proc, &exit_code)) { if (exit_code != NO_EXIT_CODE) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "\"%s\" returned with value %" PRIexitcode, name, exit_code); } else { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "\"%s\" exited unexpectedly", name); } return -1; } return 0; } static void send_keycode(enum android_keycode keycode, const char *name) { // send DOWN event struct control_event control_event = { .type = CONTROL_EVENT_TYPE_KEYCODE, .keycode_event = { .action = AKEY_EVENT_ACTION_DOWN, .keycode = keycode, .metastate = 0, }, }; if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send %s (DOWN)", name); return; } // send UP event control_event.keycode_event.action = AKEY_EVENT_ACTION_UP; if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send %s (UP)", name); } } static inline void action_home(void) { send_keycode(AKEYCODE_HOME, "HOME"); } static inline void action_back(void) { send_keycode(AKEYCODE_BACK, "BACK"); } static inline void action_app_switch(void) { send_keycode(AKEYCODE_APP_SWITCH, "APP_SWITCH"); } static inline void action_power(void) { send_keycode(AKEYCODE_POWER, "POWER"); } static inline void action_volume_up(void) { send_keycode(AKEYCODE_VOLUME_UP, "VOLUME_UP"); } static inline void action_volume_down(void) { send_keycode(AKEYCODE_VOLUME_DOWN, "VOLUME_DOWN"); } static void turn_screen_on(void) { struct control_event control_event = { .type = CONTROL_EVENT_TYPE_COMMAND, .command_event = { .action = CONTROL_EVENT_COMMAND_SCREEN_ON, }, }; if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot turn screen on"); } } static SDL_bool handle_new_frame(void) { mutex_lock(frames.mutex); AVFrame *frame = frames.rendering_frame; frames.rendering_frame_consumed = SDL_TRUE; #ifndef SKIP_FRAMES // if SKIP_FRAMES is disabled, then notify the decoder the current frame is // consumed, so that it may push a new one cond_signal(frames.rendering_frame_consumed_cond); #endif struct size current_frame_size = {frame->width, frame->height}; if (!prepare_for_frame(window, renderer, &texture, frame_size, current_frame_size)) { return SDL_FALSE; } frame_size = current_frame_size; update_texture(frame, texture); mutex_unlock(frames.mutex); render(renderer, texture); return SDL_TRUE; } static SDL_bool is_ctrl_down(void) { const Uint8 *state = SDL_GetKeyboardState(NULL); return state[SDL_SCANCODE_LCTRL] || state[SDL_SCANCODE_RCTRL]; } static void handle_text_input(const SDL_TextInputEvent *event) { if (is_ctrl_down()) { switch (event->text[0]) { case '+': action_volume_up(); break; case '-': action_volume_down(); break; } return; } struct control_event control_event; control_event.type = CONTROL_EVENT_TYPE_TEXT; strncpy(control_event.text_event.text, event->text, TEXT_MAX_LENGTH); control_event.text_event.text[TEXT_MAX_LENGTH] = '\0'; if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send text event"); } } static void handle_key(const SDL_KeyboardEvent *event) { SDL_Keycode keycode = event->keysym.sym; SDL_bool ctrl = event->keysym.mod & (KMOD_LCTRL | KMOD_RCTRL); SDL_bool shift = event->keysym.mod & (KMOD_LSHIFT | KMOD_RSHIFT); SDL_bool repeat = event->repeat; // capture all Ctrl events if (ctrl) { // only consider keydown events, and ignore repeated events if (repeat || event->type != SDL_KEYDOWN) { return; } if (shift) { // currently, there is no shortcut implying SHIFT return; } switch (keycode) { case SDLK_h: action_home(); return; case SDLK_b: // fall-through case SDLK_BACKSPACE: action_back(); return; case SDLK_m: action_app_switch(); return; case SDLK_p: action_power(); return; case SDLK_f: switch_fullscreen(); return; case SDLK_x: resize_to_fit(); return; case SDLK_g: resize_to_pixel_perfect(); return; } return; } struct control_event control_event; if (input_key_from_sdl_to_android(event, &control_event)) { if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send control event"); } } } static void handle_mouse_motion(const SDL_MouseMotionEvent *event, struct size screen_size) { if (!event->state) { // do not send motion events when no button is pressed return; } struct control_event control_event; if (mouse_motion_from_sdl_to_android(event, screen_size, &control_event)) { if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send mouse motion event"); } } } static void handle_mouse_button(const SDL_MouseButtonEvent *event, struct size screen_size) { if (event->button == SDL_BUTTON_RIGHT) { turn_screen_on(); return; }; struct control_event control_event; if (mouse_button_from_sdl_to_android(event, screen_size, &control_event)) { if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send mouse button event"); } } } static void handle_mouse_wheel(const SDL_MouseWheelEvent *event, struct position position) { struct control_event control_event; if (mouse_wheel_from_sdl_to_android(event, position, &control_event)) { if (!controller_push_event(&controller, &control_event)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Cannot send wheel button event"); } } } static void event_loop(void) { SDL_Event event; while (SDL_WaitEvent(&event)) { switch (event.type) { case EVENT_DECODER_STOPPED: SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "Video decoder stopped"); return; case SDL_QUIT: SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "User requested to quit"); return; case EVENT_NEW_FRAME: if (!handle_new_frame()) { return; } texture_empty = SDL_FALSE; count_frame(); // display fps for debug break; case SDL_WINDOWEVENT: switch (event.window.event) { case SDL_WINDOWEVENT_EXPOSED: case SDL_WINDOWEVENT_SIZE_CHANGED: render(renderer, texture_empty ? NULL : texture); break; } break; case SDL_TEXTINPUT: { handle_text_input(&event.text); break; } case SDL_KEYDOWN: case SDL_KEYUP: handle_key(&event.key); break; case SDL_MOUSEMOTION: handle_mouse_motion(&event.motion, frame_size); break; case SDL_MOUSEWHEEL: { struct position position = { .screen_size = frame_size, .point = get_mouse_point(), }; handle_mouse_wheel(&event.wheel, position); break; } case SDL_MOUSEBUTTONDOWN: case SDL_MOUSEBUTTONUP: { handle_mouse_button(&event.button, frame_size); break; } } } } SDL_bool scrcpy(const char *serial, Uint16 local_port, Uint16 max_size, Uint32 bit_rate) { SDL_bool ret = SDL_TRUE; process_t push_proc = push_server(serial); if (wait_for_success(push_proc, "adb push")) { return SDL_FALSE; } process_t reverse_tunnel_proc = enable_tunnel(serial, local_port); if (wait_for_success(reverse_tunnel_proc, "adb reverse")) { return SDL_FALSE; } TCPsocket server_socket = listen_on_port(local_port); if (!server_socket) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Could not open video socket"); goto screen_finally_adb_reverse_remove; } // server will connect to our socket process_t server = start_server(serial, max_size, bit_rate); if (server == PROCESS_NONE) { ret = SDL_FALSE; SDLNet_TCP_Close(server_socket); goto screen_finally_adb_reverse_remove; } // to reduce startup time, we could be tempted to init other stuff before blocking here // but we should not block after SDL_Init since it handles the signals (Ctrl+C) in its // event loop: blocking could lead to deadlock TCPsocket device_socket = blocking_accept(server_socket); // we don't need the server socket anymore SDLNet_TCP_Close(server_socket); if (!device_socket) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Could not accept video socket: %s", SDL_GetError()); ret = SDL_FALSE; stop_server(server); goto screen_finally_adb_reverse_remove; } char device_name[DEVICE_NAME_FIELD_LENGTH]; // screenrecord does not send frames when the screen content does not change // therefore, we transmit the screen size before the video stream, to be able // to init the window immediately if (!read_initial_device_info(device_socket, device_name, &frame_size)) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Could not retrieve initial screen size"); ret = SDL_FALSE; SDLNet_TCP_Close(device_socket); stop_server(server); goto screen_finally_adb_reverse_remove; } if (!frames_init(&frames)) { ret = SDL_FALSE; SDLNet_TCP_Close(device_socket); stop_server(server); goto screen_finally_adb_reverse_remove; } decoder.frames = &frames; decoder.video_socket = device_socket; // now we consumed the header values, the socket receives the video stream // start the decoder if (!decoder_start(&decoder)) { ret = SDL_FALSE; SDLNet_TCP_Close(device_socket); stop_server(server); goto screen_finally_destroy_frames; } if (!controller_init(&controller, device_socket)) { ret = SDL_FALSE; SDLNet_TCP_Close(device_socket); stop_server(server); goto screen_finally_stop_decoder; } if (!controller_start(&controller)) { ret = SDL_FALSE; SDLNet_TCP_Close(device_socket); stop_server(server); goto screen_finally_destroy_controller; } if (SDL_Init(SDL_INIT_VIDEO)) { SDL_LogCritical(SDL_LOG_CATEGORY_APPLICATION, "Could not initialize SDL: %s", SDL_GetError()); ret = SDL_FALSE; goto screen_finally_stop_and_join_controller; } // FIXME it may crash in SDL_Quit in i965_dri.so // As a workaround, do not call SDL_Quit() (we are exiting anyway). // atexit(SDL_Quit); // Bilinear resizing if (!SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "1")) { SDL_LogWarn(SDL_LOG_CATEGORY_VIDEO, "Could not enable bilinear filtering"); } #if SDL_VERSION_ATLEAST(2, 0, 5) // Handle a click to gain focus as any other click if (!SDL_SetHint(SDL_HINT_MOUSE_FOCUS_CLICKTHROUGH, "1")) { SDL_LogWarn(SDL_LOG_CATEGORY_VIDEO, "Could not enable mouse focus clickthrough"); } #endif struct size window_size = get_initial_optimal_size(frame_size); window = SDL_CreateWindow(device_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, window_size.width, window_size.height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE); if (!window) { SDL_LogCritical(SDL_LOG_CATEGORY_SYSTEM, "Could not create window: %s", SDL_GetError()); ret = SDL_FALSE; goto screen_finally_stop_decoder; } renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED); if (!renderer) { SDL_LogCritical(SDL_LOG_CATEGORY_RENDER, "Could not create renderer: %s", SDL_GetError()); ret = SDL_FALSE; goto screen_finally_destroy_window; } if (SDL_RenderSetLogicalSize(renderer, frame_size.width, frame_size.height)) { SDL_LogError(SDL_LOG_CATEGORY_RENDER, "Could not set renderer logical size: %s", SDL_GetError()); ret = SDL_FALSE; goto screen_finally_destroy_renderer; } SDL_Surface *icon = read_xpm(icon_xpm); if (!icon) { SDL_LogError(SDL_LOG_CATEGORY_SYSTEM, "Could not load icon: %s", SDL_GetError()); ret = SDL_FALSE; goto screen_finally_destroy_renderer; } SDL_SetWindowIcon(window, icon); SDL_FreeSurface(icon); SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Initial texture: %" PRIu16 "x%" PRIu16, frame_size.width, frame_size.height); texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_YV12, SDL_TEXTUREACCESS_STREAMING, frame_size.width, frame_size.height); if (!texture) { SDL_LogCritical(SDL_LOG_CATEGORY_RENDER, "Could not create texture: %s", SDL_GetError()); ret = SDL_FALSE; goto screen_finally_destroy_renderer; } SDL_RenderClear(renderer); SDL_RenderPresent(renderer); event_loop(); SDL_LogDebug(SDL_LOG_CATEGORY_APPLICATION, "quit..."); SDL_DestroyTexture(texture); screen_finally_destroy_renderer: // FIXME it may crash at exit if we destroy the renderer or the window, // with the exact same stack trace as . // As a workaround, leak the renderer and the window (we are exiting anyway). //SDL_DestroyRenderer(renderer); screen_finally_destroy_window: //SDL_DestroyWindow(window); // at least we hide it SDL_HideWindow(window); screen_finally_stop_and_join_controller: controller_stop(&controller); controller_join(&controller); screen_finally_destroy_controller: controller_destroy(&controller); screen_finally_stop_decoder: SDLNet_TCP_Close(device_socket); // let the server some time to print any exception trace before killing it struct timespec timespec = { .tv_sec = 0, .tv_nsec = 100000000, // 100ms }; nanosleep(×pec, NULL); // ignore error // kill the server before decoder_join() to wake up the decoder stop_server(server); decoder_join(&decoder); screen_finally_destroy_frames: frames_destroy(&frames); screen_finally_adb_reverse_remove: { process_t remove = disable_tunnel(serial); if (remove != PROCESS_NONE) { // ignore failure cmd_simple_wait(remove, NULL); } } return ret; }