I’m running on Windows 11 x64 23H2 with Visual Studio Enterprise 2022.
I am trying to hook MessageBox
in C++ to change its Background color to #457B9D
and change its foreground color to White. Also, I want to change button text and button border to White and background color to transparent.
After that, I will use this in my C# .NET 8.0 Windows Forms application, because I don’t want to create a new form for MessageBox
, because our project has grown so complex that fixing the code in each Forms is time consuming (I have a total of 90 forms and 75 user controls).
Here is my code:
framework.h
#pragma once
#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers
// Windows Header Files
#include <windows.h>
#include <stdlib.h>
pch.h
// pch.h: This is a precompiled header file.
// Files listed below are compiled only once, improving build performance for future builds.
// This also affects IntelliSense performance, including code completion and many code browsing features.
// However, files listed here are ALL re-compiled if any one of them is updated between builds.
// Do not add files here that you will be updating frequently as this negates the performance advantage.
#ifndef PCH_H
#define PCH_H
// add headers that you want to pre-compile here
#include "framework.h"
#endif //PCH_H
pch.cpp
// pch.cpp: source file corresponding to the pre-compiled header
#include "pch.h"
// When you are using pre-compiled headers, this source file is necessary for compilation to succeed.
dllmain.cpp
// dllmain.cpp : Defines the entry point for the DLL application.
#include "pch.h"
extern "C" __declspec(dllexport) void HookMessageBoxW();
extern "C" __declspec(dllexport) void UnhookMessageBoxW();
const COLORREF bgColor = RGB(69, 123, 157); // #457B9D
const COLORREF textColor = RGB(255, 255, 255); // White
HHOOK hHook = NULL;
WNDPROC oldButtonProc = NULL;
// Button subclass procedure
LRESULT CALLBACK ButtonSubclassProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) {
try {
switch (uMsg) {
case WM_PAINT: {
PAINTSTRUCT ps;
HDC hdc = BeginPaint(hWnd, &ps);
RECT rect;
GetClientRect(hWnd, &rect);
// Create and use a brush for background color
HBRUSH hBrush = CreateSolidBrush(bgColor);
FillRect(hdc, &rect, hBrush);
DeleteObject(hBrush); // Delete the brush to avoid resource leaks
SetTextColor(hdc, textColor);
SetBkMode(hdc, TRANSPARENT);
// Draw the text on the button
WCHAR text[512];
GetWindowText(hWnd, text, 512);
DrawText(hdc, text, -1, &rect, DT_CENTER | DT_VCENTER | DT_SINGLELINE);
// Draw a white border around the button
HPEN hPen = CreatePen(PS_SOLID, 2, textColor);
HGDIOBJ oldPen = SelectObject(hdc, hPen);
Rectangle(hdc, rect.left, rect.top, rect.right, rect.bottom);
SelectObject(hdc, oldPen);
DeleteObject(hPen); // Delete the pen to avoid resource leaks
EndPaint(hWnd, &ps);
return 0;
}
default:
break;
}
}
catch (...) {
// Log the exception or handle it accordingly
return CallWindowProc(oldButtonProc, hWnd, uMsg, wParam, lParam);
}
// Default processing for other messages
return CallWindowProc(oldButtonProc, hWnd, uMsg, wParam, lParam);
}
// MessageBox subclass procedure
LRESULT CALLBACK MessageBoxSubclassProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) {
try {
HDC hdcStatic;
HBRUSH hBrush = CreateSolidBrush(bgColor);
switch (uMsg) {
case WM_CTLCOLORDLG:
case WM_CTLCOLORSTATIC:
case WM_CTLCOLORBTN:
hdcStatic = (HDC)wParam;
SetTextColor(hdcStatic, textColor);
SetBkColor(hdcStatic, bgColor);
DeleteObject(hBrush); // Make sure to delete the brush after use
return (LRESULT)hBrush;
case WM_INITDIALOG: {
HWND hButton = GetDlgItem(hWnd, IDOK);
if (hButton) {
oldButtonProc = (WNDPROC)SetWindowLongPtr(hButton, GWLP_WNDPROC, (LONG_PTR)ButtonSubclassProc);
}
hButton = GetDlgItem(hWnd, IDCANCEL);
if (hButton) {
SetWindowLongPtr(hButton, GWLP_WNDPROC, (LONG_PTR)ButtonSubclassProc);
}
break;
}
case WM_DESTROY:
SetWindowLongPtr(hWnd, GWLP_WNDPROC, (LONG_PTR)GetWindowLongPtr(hWnd, GWLP_USERDATA));
SetWindowLongPtr(hWnd, GWLP_USERDATA, 0);
break;
default:
break;
}
DeleteObject(hBrush); // Delete the brush before returning
}
catch (...) {
// Handle any exceptions to prevent crashes
return DefWindowProc(hWnd, uMsg, wParam, lParam);
}
// Default processing
return CallWindowProc((WNDPROC)GetWindowLongPtr(hWnd, GWLP_USERDATA), hWnd, uMsg, wParam, lParam);
}
// Hook procedure to capture MessageBox creation
LRESULT CALLBACK CBTProc(int nCode, WPARAM wParam, LPARAM lParam) {
try {
if (nCode == HCBT_CREATEWND) {
LPCREATESTRUCT lpcs = ((LPCBT_CREATEWND)lParam)->lpcs;
// Check if it's a MessageBox
if (lpcs->lpszClass && wcscmp(lpcs->lpszClass, L"#32770") == 0) {
HWND hWnd = (HWND)wParam;
SetWindowLongPtr(hWnd, GWLP_USERDATA, SetWindowLongPtr(hWnd, GWLP_WNDPROC, (LONG_PTR)MessageBoxSubclassProc));
}
}
}
catch (...) {
// Handle the exception gracefully
return CallNextHookEx(NULL, nCode, wParam, lParam);
}
return CallNextHookEx(hHook, nCode, wParam, lParam);
}
// Exported function to hook MessageBoxW
extern "C" __declspec(dllexport) void HookMessageBoxW() {
hHook = SetWindowsHookEx(WH_CBT, CBTProc, nullptr, GetCurrentThreadId());
}
// Exported function to unhook MessageBoxW
extern "C" __declspec(dllexport) void UnhookMessageBoxW() {
if (hHook) {
UnhookWindowsHookEx(hHook);
hHook = nullptr;
}
}
Program.cs
using OSVersionExtension;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using System.Windows.Forms;
namespace Education
{
internal static class Program
{
[DllImport("Win32.dll", CallingConvention = CallingConvention.Cdecl)]
public static extern void HookMessageBoxW();
[DllImport("Win32.dll", CallingConvention = CallingConvention.Cdecl)]
public static extern void UnhookMessageBoxW();
/// <summary>
/// The main entry point for the application.
/// </summary>
[STAThread]
[System.Runtime.InteropServices.DllImport("user32.dll")]
private static extern bool SetProcessDPIAware();
static void Main()
{
HookMessageBoxW();
ApplicationConfiguration.Initialize();
//SetProcessDPIAware();
// Check Operating System section
//
// We only have rounded corners (actually is uDWM hack) on Windows 11
// On earlier OS version, we need to apply our custom rounded corner
// that defined in EllipseControl.cs
//
// To check Windows version, we use OSCheckExt from NuGet package
// manager
//
// So, we have these cases:
//
// Case 1: If users have Windows 11: Let's use native uDWM hack (inside
// dwmapi.dll) and opt in system rounded corners
//
// Case 2: If users doesn't have Windows 11: We need to create
// custom interface to enable rounded corners that defined
// in EllipseControl.cs then enable them in Form1.cs
//
// Note that on Windows Server 2022, we still doesn't have uDWM hack,
// actually uDWM hack exists only on Windows 11. So if we detected
// Windows Server Edition, we have to use our custom rounded corners
// defined in EllipseControl.cs to enable rounded corners effect
//
// 9/3/2024
OSVersionExtension.OperatingSystem osFetchData = OSVersion.GetOperatingSystem();
// Windows 11 detected
if (osFetchData == OSVersionExtension.OperatingSystem.Windows11)
{
Application.Run(new Education_MainForm(true));
}
else
{
Application.Run(new Education_MainForm(false));
}
}
}
}
When I tried to run in Debugging mode, none of useful:
I tried to figure out what is happening using System Informer, then I can see WerFault.exe is running because my application crashed:
When I tried to inspect what is happening, I get this:
It says:
An unhandled exception was encountered during a user callback.
But I can’t find anything useful when debugging, that make me very confusing.
UPDATE:
After enabling debug native code, I can see the error is:
Exception thrown at 0x00007FFD2766F7FA (ucrtbased.dll) in Education.exe: 0xC0000005: Access violation reading location 0x0000000000008002.
at line:
if (lpcs->lpszClass && wcscmp(lpcs->lpszClass, L"#32770") == 0)
2
Answers
After 3 days of debugging (including trying all the methods I came up with: Bounds checking, null pointer checking, reference checking,...) but still stuck at Access Violation, I found a solution that seems more perfect than 1 hook. However, it will require a significant amount of manipulation to get it working, but I think it's worth it, especially when it works perfectly.
The steps will be quite long, but I will try to be as detailed as possible. But in short, I used XMessageBox, a reverse-engineered implementation of the Windows MessageBox API that dates back to the Windows XP and Windows Vista era and still works today, luckily, Microsoft probably kept the MessageBox API intact to this day.
I hope that help for the people who have the similar problem with me. If you still ask for the solution to hook
MessageBox
API: Sorry, you can't do it anymoreHere are the details:
vs6Release
folder and launchXMsgBoxTest.exe
. This will be where we startGallery...
Click on
Custom Icon + Custom Colors
, this is what we needThis is what we need. Now, we need to figure out how to apply it.... Let's look at this box, this is the code that we need to be able to display the MessageBox as shown in the above sample
Now, we need to SAVE this code, for future use. Now, go back to Visual Studio, create a new C++ DLL MFC project:
Give it any name you like, but make sure it is in the same solution. After creation, we will get this:
Now, come back with XMessageBox folder, navigate to
src
directory. You should see this:Now we need to grab neccessary files. There are these file is required:
HEADER FILES
XHyperLink.h
XMessageBox.h
XTrace.h
SOURCE FILES
XHyperLink.cpp
XMessageBox.cpp
Here are the files we need:
Copy them to our C++ MFC DLL Project source code directory, after that, we will get this:
Come back to Visual Studio, right click on
Header Files
in C++ MFC DLL Project, then selectAdd > Existing Item...
then select all of our HEADER FILES
After that, do the same thing with Source Files. After that, you will get this project structure like:
Now, just a thing that not important, go to
Win32MFC.cpp
(note that name file will change, depend on your project name), then comment all of the code inside them, then add#include "pch.h"
on this file. This is very important:Do the samething with header file:
Now come back to
src
directory of XMessageBox, open fileStdAfx.h
with your favourite editor, I will use notepad:Copy all of codes to replace current contents of your
framework.h
:Now, open your
pch.h
files. Then add these code after#include "framework.h"
:Your file should like this:
Now, open
XMessageBox.h
files, then find this:This is the main function we need to call to display our custom MessageBox. We need to export them too. Just replace the function to:
Now go to
XMessageBox.cpp
, findXMessageBox()
. Here is the thing you should see:Replace this function to:
Then, replace
XHyperLink.cpp
,XMessageBox.cpp
line#include "StdAfx.h"
or#include "stdafx.h"
to#include "pch.h"
:Now press Ctrl + Shift + B to build project. After that, you will get this in
<Your Solution Directory>x64Release
(Note thatx64
andRelease
will depend on your build configuration):You will get your DLL like:
Now let's inspect to the DLL to see the function was exported. I will use
PE Viewer
bundled withSystem Informer
, you can use another tool, such asCFF Explorer
. Open your DLL inside your inspector, then you will get this:Click on
Exports
tab:and you will see something like:
If not, please check if you have
__declspec(dllexport)
andextern "C"
. Note thatextern "C"
is required to tell to the computer that it should not be decorated the function. If not, you will see theUndecorated name
have value. In that case, make sure you have already putextern "C"
before__declspec(dllexport)
. Rebuild your project again after made changes.After you can sure that your DLL function was exported correctly, back to Visual Studio, then in the same solution, create a new
C++/CLI DLL Project
.If you don't know what
C++/CLI DLL Project
is:Now, in Visual Studio, create an
C++/CLI DLL Project
, give them with any name you like, then pressCreate
:Depend on your project was written on .NET or .NET Framework, you must choice it correct. My application written on .NET, so I will choice .NET
Now, look back to
XMessageBox()
declaration:We can see parameters required
XMSGBOXPARAMS
. So where isXMSGBOXPARAMS
? They defined in the same fileXMessageBox.h
:So we must need
XMSGBOXPARAMS
structure to make our DLL Wrapper working. The easy way is just copyXMessageBox.h
to our project directory, add to Header Files section and comment out the function we don't need to use in our wrapper and also prevent the errors.Do the same thing to copy
XMessageBox.h
to project and add to Header files, here is the final result:Now go to
XMessageBox.h
in your wrapper project, then comment this section:Your file now should like:
Now create new C++ file in Source Files section. Given it any name you like. Then use this code:
Now, build your project. Now you should see two DLLs, like this:
If you wonder what
ijwhost.dll
is:Now we are almost done! Come back to your C# project, right click to your project (It has C# logo), then select
Add > Project reference...
and you will see your
C++/CLI DLL
Wrapper:Check it, then select OK to add reference.
Now, we can test our program:
Program.cs
Build your project, but don't run your application
Copy
Win32MFC.dll
to your directory that contains your C# application in EXE file. After that, your directory will look like:Here is the result you should see:
Good luck!
Per the
CREATESTRUCT
documentation:Your code assumes the
lpszClass
is always a pointer to a null-terminated string and never an atom, which system dialogs do use.You can use the
IS_INTRESOURCE()
macro to differentiate between them, eg: