use int2f to detect fullscreen dosboxes in protected mode

4 files changed, 181 insertions, 3 deletions

diff --git a/int2fwin.h b/int2fwin.h
new file mode 100644
index 0000000..8d79211
--- /dev/null
+++ b/int2fwin.h
@@ -0,0 +1,55 @@
+/*
+ * VBMouse - Interface to the Windows int 2Fh services
+ * Copyright (C) 2022 Javier S. Pedro
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef INT2FWIN_H
+#define INT2FWIN_H
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <dos.h>
+
+typedef void (__far *LPFN)(void);
+
+enum int2f_functions
+{
+	INT2F_NOTIFY_BACKGROUND_SWITCH = 0x4001,
+	INT2F_NOTIFY_FOREGROUND_SWITCH = 0x4002
+};
+
+static bool windows_386_enhanced_mode(void);
+#pragma aux windows_386_enhanced_mode = \
+	"mov ax, 0x1600" \
+	"int 0x2F" \
+	"test al, 0x7F" /* return value is either 0x80 or 0x00 if win386 is not running */ \
+	"setnz al" \
+	__value [al] \
+	__modify [ax]
+
+static inline void hook_int2f(LPFN *prev, LPFN new)
+{
+	*prev = _dos_getvect(0x2F);
+	_dos_setvect(0x2F, new);
+}
+
+static inline void unhook_int2f(LPFN prev)
+{
+	_dos_setvect(0x2F, prev);
+}
+
+#endif
diff --git a/mousew16.c b/mousew16.c
index c017133..f2e22ab 100644
--- a/mousew16.c
+++ b/mousew16.c
@@ -22,10 +22,14 @@
 #include "vbox.h"
 #include "vboxdev.h"
 #include "ps2.h"
+#include "int2fwin.h"
 #include "mousew16.h"
 
-/** If this is 0, this should behave like a plain PS/2 mouse driver. */
+/** If 1, actually call VBox services.
+ *  Thus, if 0, this should behave like a plain PS/2 mouse driver. */
 #define ENABLE_VBOX 1
+/** If 1, hook int2f to detect fullscreen DOSBoxes and auto-disable this driver. */
+#define HOOK_INT2F  1
 
 /** Logging through the virtualbox backdoor. */
 #define log(...) vbox_logs(__VA_ARGS__)
@@ -40,9 +44,13 @@ enum {
 	MOUSEFLAGS_ENABLED      = 1 << 0,
 	MOUSEFLAGS_HAS_VBOX     = 1 << 1,
 	MOUSEFLAGS_VBOX_ENABLED = 1 << 2,
+	MOUSEFLAGS_HAS_WIN386   = 1 << 3,
+	MOUSEFLAGS_INT2F_HOOKED = 1 << 4
 };
 /** Last received pressed button status (to compare and see which buttons have been pressed). */
 static unsigned char mousebtnstatus;
+/** Existing interrupt2f handler. */
+static LPFN prev_int2f_handler;
 
 /* This is how events are delivered to Windows */
 
@@ -104,6 +112,74 @@ static void FAR ps2_mouse_callback(uint8_t status, uint8_t x, uint8_t y, uint8_t
 	}
 }
 
+#if HOOK_INT2F
+
+static void display_switch_handler(int function)
+#pragma aux display_switch_handler parm caller [ax] modify [ax bx cx dx si di]
+{
+	if (!(mouseflags & MOUSEFLAGS_ENABLED) || !(mouseflags & MOUSEFLAGS_VBOX_ENABLED)) {
+		return;
+	}
+
+	switch (function) {
+	case INT2F_NOTIFY_BACKGROUND_SWITCH:
+		vbox_logs("Going background\n");
+		vbox_set_mouse_locked(false);
+		break;
+	case INT2F_NOTIFY_FOREGROUND_SWITCH:
+		vbox_logs("Going foreground\n");
+		vbox_set_mouse_locked(true);
+		break;
+	}
+}
+
+/** Interrupt 2F handler, which will be called on some Windows 386 mode events.
+ * This is more complicated than it should be becaused we have to fetch our DS
+ * without clobbering any registers whatsoever, and chain back to the previous handler.
+ * @todo OpenWatcom 2.x insists on calling GETDS from a different code segment, so we can't use __interrupt. */
+static void __declspec(naked) __far int2f_handler(void)
+{
+	_asm {
+		; Preserve data segment
+		push ds
+
+		; Load our data segment
+		push ax
+		mov ax, SEG prev_int2f_handler ; Let's hope that Windows relocates segments with interrupts disabled
+		mov ds, ax
+		pop ax
+
+		; Check functions we are interested in hooking
+		cmp ax, 0x4001  ; Notify Background Switch
+		je handle_it
+		cmp ax, 0x4002  ; Notify Foreground Switch
+		je handle_it
+
+		; Otherwise directly jump to next handler
+		jmp next_handler
+
+	handle_it:
+		pushad ; Save and restore 32-bit registers, we may clobber them from C
+		call display_switch_handler
+		popad
+
+	next_handler:
+		; Store the address of the previous handler
+		push dword ptr [prev_int2f_handler]
+
+		; Restore original data segment without touching the stack,
+		; since we want to keep the prev handler address at the top
+		push bp
+		mov bp, sp
+		mov ds, [bp + 6]  ; Stack looks like 0: bp, 2: prev_int2f_handler, 6: ds
+		pop bp
+
+		retf 2
+	}
+}
+
+#endif /* HOOK_INT2F */
+
 #pragma code_seg ()
 
 /* Driver exported functions. */
@@ -119,6 +195,13 @@ BOOL FAR PASCAL LibMain(HINSTANCE hInstance, WORD wDataSegment,
 {
 	// We are not going to bother checking whether a PS2 mouse exists and just assume it does
 
+#if HOOK_INT2F
+	// Check now whether we are running under protected mode windows
+	if (windows_386_enhanced_mode()) {
+		mouseflags |= MOUSEFLAGS_HAS_WIN386;
+	}
+#endif
+
 #if ENABLE_VBOX
 	// However we will check whether VirtualBox exists:
 	if (vbox_init() == 0) {
@@ -187,6 +270,16 @@ VOID FAR PASCAL Enable(LPFN_MOUSEEVENT lpEventProc)
 			mouseflags |= MOUSEFLAGS_VBOX_ENABLED;
 		}
 #endif
+
+#if HOOK_INT2F
+		if ((mouseflags & MOUSEFLAGS_HAS_WIN386) && (mouseflags & MOUSEFLAGS_VBOX_ENABLED)) {
+			cli();
+			hook_int2f(&prev_int2f_handler, int2f_handler);
+			sti();
+			vbox_logs("int2F hooked!\n");
+			mouseflags |= MOUSEFLAGS_INT2F_HOOKED;
+		}
+#endif
 	}
 }
 
@@ -194,6 +287,16 @@ VOID FAR PASCAL Enable(LPFN_MOUSEEVENT lpEventProc)
 VOID FAR PASCAL Disable(VOID)
 {
 	if (mouseflags & MOUSEFLAGS_ENABLED) {
+#if HOOK_INT2F
+		if (mouseflags & MOUSEFLAGS_INT2F_HOOKED) {
+			cli();
+			unhook_int2f(prev_int2f_handler);
+			sti();
+			vbox_logs("int2F unhooked!\n");
+			mouseflags &= ~MOUSEFLAGS_INT2F_HOOKED;
+		}
+#endif
+
 		ps2_enable(false);
 		ps2_set_callback(NULL);
 		vbox_logs("PS2 Disabled!\n");
@@ -214,6 +317,6 @@ VOID FAR PASCAL Disable(VOID)
 /** Called by Window to retrieve the interrupt vector number used by this driver, or -1. */
 int FAR PASCAL MouseGetIntVect(VOID)
 {
-	return -1;
+	return 0x74; /* This corresponds to IRQ12, the PS/2 IRQ. At least in VirtualBox. */
 }
 
diff --git a/vbox.c b/vbox.c
index 6278d6b..38673e0 100644
--- a/vbox.c
+++ b/vbox.c
@@ -270,8 +270,26 @@ int vbox_get_mouse(bool *abs, uint16_t *xpos, uint16_t *ypos)
 
 #pragma code_seg ( "CALLBACKS" )
 
-/** This is a version of vbox_get_mouse() that does not call any other functions,
+/** This is a version of vbox_set_mouse() that does not call any other functions,
   * and may be called inside an interrupt handler.  */
+int vbox_set_mouse_locked(bool enable)
+{
+	VMMDevReqMouseStatus *req = pBuf;
+
+	req->header.size = sizeof(VMMDevReqMouseStatus);
+	req->header.version = VMMDEV_REQUEST_HEADER_VERSION;
+	req->header.requestType = VMMDevReq_SetMouseStatus;
+	req->header.rc = -1;
+	req->mouseFeatures = enable ? VMMDEV_MOUSE_GUEST_CAN_ABSOLUTE : 0;
+	req->pointerXPos = 0;
+	req->pointerYPos = 0;
+
+	vbox_send_request(bufdds.physicalAddress);
+
+	return req->header.rc;
+}
+
+/** Likewise for vbox_get_mouse() */
 int vbox_get_mouse_locked(bool *abs, uint16_t *xpos, uint16_t *ypos)
 {
 	VMMDevReqMouseStatus *req = pBuf;
diff --git a/vbox.h b/vbox.h
index 8fbd80e..2b9c826 100644
--- a/vbox.h
+++ b/vbox.h
@@ -68,6 +68,8 @@ extern int vbox_get_mouse(bool *abs, uint16_t *xpos, uint16_t *ypos);
 
 // In CALLBACKS segment:
 
+extern int vbox_set_mouse_locked(bool enable);
+
 extern int vbox_get_mouse_locked(bool *abs, uint16_t *xpos, uint16_t *ypos);
 
 #endif