To respond to special mouse button actions in Win32, we need to register the virtual command actions with the callback loop to detect the special mouse button action. For example, to detect forward and backward mouse button actions, we register the APPCOMMAND_BROWSER_FORWARD and APPCOMMAND_BROWSER_BACKWARD events:

LRESULT CALLBACK MainWndProc(HWND hWnd, UINT message, WPARAM wParam, LONG lParam) {
	switch (message) {
	case WM_APPCOMMAND:
		{
			int command = GET_APPCOMMAND_LPARAM(lParam);
			if(command==APPCOMMAND_BROWSER_BACKWARD) {
				//do back button action
			}
			else if(command==APPCOMMAND_BROWSER_FORWARD) {
				//do forward button action
			}
			else { return FALSE; } //no action detected, try other handlers
			return TRUE; //action handled
		}
	}
}

Should the WM_APPCOMMAND options not work, you might want to look into the VK_* event codes.