From 79ced09dcf83f537d7bde88df586b37db7bb88b0 Mon Sep 17 00:00:00 2001
From: Peter Repukat <orly.alia5@gmail.com>
Date: Sun, 12 Feb 2017 22:40:18 +0100
Subject: [PATCH] SteamTarget: Better way to detect overlay opening / closing

Found out GameOverlayRenderer likes to post very specific messages
Now, instead of memory reading some value from some address, which changes in updates, examine the messagequeue of the process to get overlay status
---
 SteamTarget/SteamTargetRenderer.cpp | 57 ++++++++++++++++++++---------
 SteamTarget/SteamTargetRenderer.h   | 17 +++++----
 2 files changed, 50 insertions(+), 24 deletions(-)

diff --git a/SteamTarget/SteamTargetRenderer.cpp b/SteamTarget/SteamTargetRenderer.cpp
index bae3a6b..5aeaa54 100644
--- a/SteamTarget/SteamTargetRenderer.cpp
+++ b/SteamTarget/SteamTargetRenderer.cpp
@@ -15,6 +15,8 @@ limitations under the License.
 */
 #include "SteamTargetRenderer.h"
 
+std::atomic<bool> SteamTargetRenderer::overlayOpen = false;
+HHOOK SteamTargetRenderer::hook = nullptr;
 
 SteamTargetRenderer::SteamTargetRenderer(int& argc, char** argv) : QApplication(argc, argv)
 {
@@ -61,10 +63,23 @@ SteamTargetRenderer::SteamTargetRenderer(int& argc, char** argv) : QApplication(
 		controllerThread.run();
 
 	QTimer::singleShot(2000, this, &SteamTargetRenderer::launchApp); // lets steam do its thing
+
+	if (hmodGameOverlayRenderer != nullptr)
+	{
+		//Hook MessageQueue to detect if overlay gets opened / closed
+		//Steam Posts a Message with 0x14FA / 0x14F7 when the overlay gets opened / closed
+		hook = SetWindowsHookEx(WH_GETMESSAGE, HookCallback, nullptr, GetCurrentThreadId());
+	}
 }
 
 SteamTargetRenderer::~SteamTargetRenderer()
 {	
+
+	if (hmodGameOverlayRenderer != nullptr)
+	{
+		UnhookWindowsHookEx(hook);
+	}
+
 	renderThread.join();
 	if (controllerThread.isRunning())
 		controllerThread.stop();
@@ -81,13 +96,13 @@ void SteamTargetRenderer::stop()
 	QApplication::exit(0);
 }
 
+
 void SteamTargetRenderer::RunSfWindowLoop()
 {
 	if (!bRunLoop)
 		return;
 	sfWindow.setActive(true);
 
-	sf::Clock reCheckControllerTimer;
 	bool focusSwitchNeeded = true;
 
 	if (bDrawOverlay)
@@ -137,9 +152,8 @@ void SteamTargetRenderer::RunSfWindowLoop()
 
 		//Dirty hack to make the steamoverlay work properly and still keep Apps Controllerconfig when closing overlay.
 		//even if hooking steam, this ensures the overlay stays working
-		if (overlayPtr != NULL)
+		if (hmodGameOverlayRenderer != NULL)
 		{
-			char overlayOpen = *reinterpret_cast<char*>(overlayPtr);
 			if (overlayOpen)
 			{
 				if (!bNeedFocusSwitch)
@@ -196,24 +210,12 @@ void SteamTargetRenderer::RunSfWindowLoop()
 
 void SteamTargetRenderer::getSteamOverlay()
 {
-	//TODO: switch to pattern scanning... this is madness.
-#ifdef _AMD64_
-	hmodGameOverlayRenderer = GetModuleHandle(L"Gameoverlayrenderer64.dll");
+	hmodGameOverlayRenderer = GetModuleHandle(overlayModuleName);
 
 	if (hmodGameOverlayRenderer != nullptr)
 	{
-		std::cout << "GameOverlayrenderer64.dll found;  Module at: 0x" << hmodGameOverlayRenderer << std::endl;
-		overlayPtr = reinterpret_cast<uint64_t*>(uint64_t(hmodGameOverlayRenderer) + 0x1365cc);
+		std::cout << overlayModuleName << " found;  Module at: 0x" << hmodGameOverlayRenderer << std::endl;
 	}
-#else
-	hmodGameOverlayRenderer = GetModuleHandle(L"Gameoverlayrenderer.dll");
-
-	if (hmodGameOverlayRenderer != nullptr)
-	{
-		std::cout << "GameOverlayrenderer.dll found;  Module at: 0x" << hmodGameOverlayRenderer << std::endl;
-		overlayPtr = reinterpret_cast<uint32_t*>(uint32_t(hmodGameOverlayRenderer) + 0xEE828);
-	}
-#endif
 }
 
 
@@ -320,6 +322,25 @@ void SteamTargetRenderer::loadLogo()
 	backgroundSprite.setPosition(sf::Vector2f(winSize.width / 2.f, winSize.height / 2.f));
 }
 
+LRESULT WINAPI SteamTargetRenderer::HookCallback(int nCode, WPARAM wParam, LPARAM lParam)
+{
+	if (nCode >= 0)
+	{
+		PMSG msg = reinterpret_cast<PMSG>(lParam);
+		if (msg->message == 0x14FA) //Posted when the overlay gets opened
+		{
+			overlayOpen = true;
+			std::cout << "Overlay Opened!\n";
+		}
+		else if (msg->message == 0x14F7 || msg->message == 512)
+		{
+			overlayOpen = false;
+			std::cout << "Overlay closed!\n";
+		}
+	}
+	return CallNextHookEx(hook, nCode, wParam, lParam);
+}
+
 void SteamTargetRenderer::launchApp()
 {
 	bool launchGame = false;
@@ -423,5 +444,7 @@ void SteamTargetRenderer::checkSharedMem()
 			exit(0);
 		}
 	}
+
+
 }
 
diff --git a/SteamTarget/SteamTargetRenderer.h b/SteamTarget/SteamTargetRenderer.h
index 9c2efa8..307bc3a 100644
--- a/SteamTarget/SteamTargetRenderer.h
+++ b/SteamTarget/SteamTargetRenderer.h
@@ -24,6 +24,7 @@ limitations under the License.
 #include <iostream>
 #include <thread>
 
+#include <QApplication>
 #include <QTimer>
 #include <QProcess>
 #include <QBuffer>
@@ -32,10 +33,9 @@ limitations under the License.
 #include <QSettings>
 #include <QCoreApplication>
 #include <QDir>
-
+#include <psapi.h>
 
 #include "VirtualControllerThread.h"
-#include <QApplication>
 
 class SteamTargetRenderer : public QApplication
 {
@@ -51,7 +51,6 @@ public:
 
 
 private:
-
 	void stop();
 	void getSteamOverlay();
 	void RunSfWindowLoop();
@@ -61,7 +60,7 @@ private:
 
 	void loadLogo();
 
-	bool bRunLoop = true;
+	std::atomic<bool> bRunLoop = true;
 
 	bool bDrawDebugEdges = false;
 	bool bDrawOverlay = true;
@@ -74,12 +73,16 @@ private:
 
 	HWND consoleHwnd;
 
-	HMODULE hmodGameOverlayRenderer;
+	HMODULE hmodGameOverlayRenderer = nullptr;
 #ifdef _AMD64_
-	uint64_t *overlayPtr = nullptr;
+	WCHAR* overlayModuleName = L"GameOverlayRenderer64.dll";
 #else
-	uint32_t *overlayPtr = nullptr;
+	WCHAR* overlayModuleName = L"GameOverlayRenderer.dll";
 #endif
+	static std::atomic<bool> overlayOpen;
+	static HHOOK hook;
+	static LRESULT WINAPI HookCallback(int nCode, WPARAM wParam, LPARAM lParam);
+
 	HWND hwForeGroundWindow = nullptr;
 	bool bNeedFocusSwitch = false;