Enhance Big Picture Mode OSK and webview input support

Adds native input event injection for webviews via IPC, improves the on-screen keyboard (OSK) UI with a blinking cursor and label, and enables seamless text entry into webview input fields. Also refines virtual cursor click handling for better compatibility with complex sites and video players.

main.js

@@ -204,6 +204,22 @@ ipcMain.on('exit-bigpicture', () => {
   exitBigPictureMode();
 });
 
+// IPC handler for sending mouse input events to webviews (used by Big Picture Mode)
+ipcMain.handle('webview-send-input-event', async (event, { webContentsId, inputEvent }) => {
+  try {
+    const { webContents: webContentsModule } = require('electron');
+    const targetWebContents = webContentsModule.fromId(webContentsId);
+    if (targetWebContents && !targetWebContents.isDestroyed()) {
+      targetWebContents.sendInputEvent(inputEvent);
+      return { success: true };
+    }
+    return { success: false, error: 'WebContents not found' };
+  } catch (err) {
+    console.error('[Main] webview-send-input-event error:', err);
+    return { success: false, error: err.message };
+  }
+});
+
 // =============================================================================
 
 

preload.js

@@ -127,7 +127,10 @@ contextBridge.exposeInMainWorld('bigPictureAPI', {
   // Exit Big Picture Mode
   exit: () => ipcRenderer.invoke('exit-bigpicture'),
   // Navigate to URL (from Big Picture Mode)
-  navigate: (url) => ipcRenderer.send('bigpicture-navigate', url)
+  navigate: (url) => ipcRenderer.send('bigpicture-navigate', url),
+  // Send input event to a webview (for virtual cursor clicks)
+  sendInputEvent: (webContentsId, inputEvent) => 
+    ipcRenderer.invoke('webview-send-input-event', { webContentsId, inputEvent })
 });
 
 // Relay context-menu commands from main to active renderer context (open new tabs etc.)

renderer/bigpicture.css

@@ -982,6 +982,22 @@ body.mouse-active {
   padding: var(--bp-spacing-lg);
 }
 
+.osk-title {
+  display: flex;
+  align-items: center;
+  gap: var(--bp-spacing-md);
+  margin-bottom: var(--bp-spacing-md);
+  color: var(--bp-accent);
+  font-size: 1rem;
+  font-weight: 600;
+  text-transform: uppercase;
+  letter-spacing: 1px;
+}
+
+.osk-title .material-symbols-outlined {
+  font-size: 1.3rem;
+}
+
 .osk-header {
   display: flex;
   align-items: center;
@@ -989,19 +1005,67 @@ body.mouse-active {
   margin-bottom: var(--bp-spacing-md);
 }
 
+.osk-input-wrapper {
+  flex: 1;
+  position: relative;
+  display: flex;
+  align-items: center;
+  background: var(--bp-bg);
+  border: 2px solid var(--bp-accent);
+  border-radius: var(--bp-radius-md);
+  box-shadow: 0 0 20px var(--bp-accent-glow);
+  overflow: hidden;
+}
+
 .osk-text-input {
   flex: 1;
   padding: var(--bp-spacing-md) var(--bp-spacing-lg);
-  background: var(--bp-bg);
+  background: transparent;
-  border: 2px solid var(--bp-border);
+  border: none;
-  border-radius: var(--bp-radius-md);
+  font-size: 1.3rem;
-  font-size: 1.2rem;
   color: var(--bp-text);
+  font-weight: 500;
+  letter-spacing: 0.5px;
+  outline: none;
 }
 
-.osk-text-input:focus {
+.osk-text-input::placeholder {
-  outline: none;
+  color: var(--bp-text-dim);
-  border-color: var(--bp-accent);
+}
+
+/* Blinking cursor that follows text */
+.osk-cursor {
+  position: absolute;
+  top: 50%;
+  transform: translateY(-50%);
+  width: 3px;
+  height: 1.5em;
+  background: var(--bp-accent);
+  border-radius: 2px;
+  animation: blink-cursor 1s step-end infinite;
+  box-shadow: 0 0 8px var(--bp-accent);
+  pointer-events: none;
+  left: var(--bp-spacing-lg);
+}
+
+/* Hidden element to measure text width */
+.osk-text-measure {
+  position: absolute;
+  visibility: hidden;
+  white-space: pre;
+  font-size: 1.3rem;
+  font-weight: 500;
+  letter-spacing: 0.5px;
+  font-family: inherit;
+}
+
+@keyframes blink-cursor {
+  0%, 100% {
+    opacity: 1;
+  }
+  50% {
+    opacity: 0;
+  }
 }
 
 .osk-close {

renderer/bigpicture.html

@@ -254,8 +254,16 @@
     <!-- On-screen keyboard (for controller input) -->
     <div id="osk-overlay" class="osk-overlay hidden">
       <div class="osk-container">
+        <div class="osk-title">
+          <span class="material-symbols-outlined">keyboard</span>
+          <span id="osk-label">Enter text</span>
+        </div>
         <div class="osk-header">
-          <input type="text" id="osk-input" class="osk-text-input" placeholder="Type with D-pad + A" readonly>
+          <div class="osk-input-wrapper">
+            <input type="text" id="osk-input" class="osk-text-input" placeholder="Your text appears here..." readonly>
+            <span id="osk-cursor" class="osk-cursor"></span>
+            <span id="osk-text-measure" class="osk-text-measure"></span>
+          </div>
           <button class="osk-close" data-focusable tabindex="0">
             <span class="material-symbols-outlined">close</span>
           </button>

renderer/bigpicture.js

@@ -73,6 +73,7 @@ const state = {
   
   // Webview for browsing
   currentWebview: null,
+  webviewContentsId: null, // For native input event injection
   webviewStack: []  // Stack of webview instances for navigation history
 };
 
@@ -827,6 +828,7 @@ function initOSK() {
 function openOSK(mode = 'search') {
   const overlay = document.getElementById('osk-overlay');
   const input = document.getElementById('osk-input');
+  const label = document.getElementById('osk-label');
   
   if (!overlay || !input) return;
   
@@ -837,6 +839,14 @@ function openOSK(mode = 'search') {
   // Clear input
   input.value = '';
   
+  // Reset cursor position
+  updateOSKCursorPosition();
+  
+  // Update label based on mode
+  if (label) {
+    label.textContent = mode === 'search' ? 'Search or enter URL' : 'Enter text';
+  }
+  
   // Update focusable elements to only include OSK keys
   updateFocusableElements();
   
@@ -855,6 +865,51 @@ function openOSK(mode = 'search') {
   }, 100);
 }
 
+/**
+ * Open OSK for typing into a focused input field in the webview
+ */
+function openOSKForWebview() {
+  const overlay = document.getElementById('osk-overlay');
+  const input = document.getElementById('osk-input');
+  const label = document.getElementById('osk-label');
+  
+  if (!overlay || !input) return;
+  
+  state.oskVisible = true;
+  state.oskMode = 'webview'; // Special mode for webview input
+  overlay.classList.remove('hidden');
+  
+  // Clear input (could optionally preserve current input value)
+  input.value = '';
+  
+  // Reset cursor position
+  updateOSKCursorPosition();
+  
+  // Update the label to indicate webview mode
+  if (label) {
+    label.textContent = 'Type your text';
+  }
+  
+  // Update focusable elements to only include OSK keys
+  updateFocusableElements();
+  
+  // Focus first key
+  setTimeout(() => {
+    const firstKey = overlay.querySelector('.osk-key');
+    if (firstKey) {
+      const index = state.focusableElements.indexOf(firstKey);
+      if (index !== -1) {
+        state.focusIndex = index;
+        focusElement(firstKey);
+      } else {
+        firstKey.focus();
+      }
+    }
+  }, 100);
+  
+  showToast('📝 Type and press Submit to enter text');
+}
+
 function closeOSK() {
   const overlay = document.getElementById('osk-overlay');
   if (!overlay) return;
@@ -873,6 +928,7 @@ function appendToOSK(char) {
   const input = document.getElementById('osk-input');
   if (input) {
     input.value += char;
+    updateOSKCursorPosition();
   }
 }
 
@@ -880,6 +936,7 @@ function backspaceOSK() {
   const input = document.getElementById('osk-input');
   if (input && input.value.length > 0) {
     input.value = input.value.slice(0, -1);
+    updateOSKCursorPosition();
     playNavSound();
   }
 }
@@ -888,23 +945,98 @@ function clearOSK() {
   const input = document.getElementById('osk-input');
   if (input) {
     input.value = '';
+    updateOSKCursorPosition();
     playNavSound();
   }
 }
 
+/**
+ * Update the blinking cursor position to follow the text
+ */
+function updateOSKCursorPosition() {
+  const input = document.getElementById('osk-input');
+  const cursor = document.getElementById('osk-cursor');
+  const measure = document.getElementById('osk-text-measure');
+  
+  if (!input || !cursor || !measure) return;
+  
+  // Copy the input text to the measure element
+  measure.textContent = input.value || '';
+  
+  // Get the text width + padding offset
+  const textWidth = measure.offsetWidth;
+  const paddingLeft = 32; // var(--bp-spacing-lg) = 32px
+  
+  // Position cursor right after the text
+  cursor.style.left = `${paddingLeft + textWidth}px`;
+}
+
 function submitOSK() {
   const input = document.getElementById('osk-input');
-  if (!input || !input.value.trim()) return;
+  if (!input) return;
   
-  const value = input.value.trim();
+  const value = input.value;
   
   if (state.oskMode === 'search') {
-    performSearch(value);
+    if (!value.trim()) return;
+    performSearch(value.trim());
+  } else if (state.oskMode === 'webview' && state.currentWebview) {
+    // Send the typed text to the webview's focused input
+    sendTextToWebview(value, true); // true = submit after setting
   }
   
   closeOSK();
 }
 
+/**
+ * Send typed text from OSK to the focused input field in webview
+ */
+function sendTextToWebview(text, submit = false) {
+  if (!state.currentWebview) return;
+  
+  try {
+    // Send the text value to the webview
+    const script = submit ? `
+      (function() {
+        const activeEl = document.activeElement;
+        if (activeEl && (activeEl.tagName === 'INPUT' || activeEl.tagName === 'TEXTAREA' || activeEl.isContentEditable)) {
+          activeEl.value = ${JSON.stringify(text)};
+          activeEl.dispatchEvent(new Event('input', { bubbles: true }));
+          activeEl.dispatchEvent(new Event('change', { bubbles: true }));
+          
+          // Trigger Enter key to submit
+          setTimeout(() => {
+            activeEl.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', keyCode: 13, bubbles: true }));
+            activeEl.dispatchEvent(new KeyboardEvent('keypress', { key: 'Enter', keyCode: 13, bubbles: true }));
+            activeEl.dispatchEvent(new KeyboardEvent('keyup', { key: 'Enter', keyCode: 13, bubbles: true }));
+            
+            // Also try form submission
+            const form = activeEl.closest('form');
+            if (form) {
+              const submitBtn = form.querySelector('button[type="submit"], input[type="submit"], button:not([type])');
+              if (submitBtn) submitBtn.click();
+            }
+          }, 50);
+        }
+      })();
+    ` : `
+      (function() {
+        const activeEl = document.activeElement;
+        if (activeEl && (activeEl.tagName === 'INPUT' || activeEl.tagName === 'TEXTAREA' || activeEl.isContentEditable)) {
+          activeEl.value = ${JSON.stringify(text)};
+          activeEl.dispatchEvent(new Event('input', { bubbles: true }));
+        }
+      })();
+    `;
+    
+    state.currentWebview.executeJavaScript(script).catch(err => {
+      console.log('[BigPicture] Send text error:', err);
+    });
+  } catch (err) {
+    console.log('[BigPicture] sendTextToWebview error:', err);
+  }
+}
+
 function handleOSKKeyboard(e) {
   if (e.key === 'Escape') {
     e.preventDefault();
@@ -1189,6 +1321,30 @@ function navigateTo(url) {
   
   container.appendChild(webview);
   state.currentWebview = webview;
+  state.webviewContentsId = null; // Will be set when webview is ready
+  
+  // Get webContentsId when webview is ready for native input events
+  webview.addEventListener('dom-ready', () => {
+    try {
+      // getWebContentsId is available on webview element
+      state.webviewContentsId = webview.getWebContentsId();
+      console.log('[BigPicture] WebContents ID:', state.webviewContentsId);
+      
+      // Inject script to detect input field focus and notify the host
+      injectInputFocusDetection(webview);
+    } catch (err) {
+      console.log('[BigPicture] Could not get webContentsId:', err);
+    }
+  });
+  
+  // Listen for IPC messages from webview (for OSK requests)
+  webview.addEventListener('ipc-message', (event) => {
+    if (event.channel === 'bigpicture-input-focused') {
+      // Input field was clicked/focused in webview - show OSK for webview input
+      console.log('[BigPicture] Input focused in webview');
+      openOSKForWebview();
+    }
+  });
   
   // Enable virtual cursor for webview interaction
   enableCursor();
@@ -1202,6 +1358,80 @@ function navigateTo(url) {
   }, 100);
 }
 
+/**
+ * Inject script to detect input focus in webview and send message to host
+ */
+function injectInputFocusDetection(webview) {
+  const script = `
+    (function() {
+      if (window.__bigPictureInputDetection) return;
+      window.__bigPictureInputDetection = true;
+      
+      // Track the last focused input
+      let lastFocusedInput = null;
+      
+      document.addEventListener('focusin', (e) => {
+        const el = e.target;
+        const isInput = el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' ||
+                       el.contentEditable === 'true' || el.isContentEditable ||
+                       el.getAttribute('role') === 'textbox' || el.getAttribute('role') === 'searchbox';
+        
+        // Check input type - exclude non-text inputs
+        if (el.tagName === 'INPUT') {
+          const type = el.type.toLowerCase();
+          if (['checkbox', 'radio', 'submit', 'button', 'image', 'file', 'hidden', 'reset', 'range', 'color'].includes(type)) {
+            return;
+          }
+        }
+        
+        if (isInput) {
+          lastFocusedInput = el;
+          // Send message to host (Big Picture Mode) to show OSK
+          try {
+            if (window.electronAPI && window.electronAPI.sendToHost) {
+              window.electronAPI.sendToHost('bigpicture-input-focused', {
+                type: el.tagName,
+                inputType: el.type || 'text',
+                value: el.value || ''
+              });
+            }
+          } catch(e) {
+            console.log('BigPicture: Could not notify input focus', e);
+          }
+        }
+      }, true);
+      
+      // Listen for text input from OSK
+      window.addEventListener('message', (e) => {
+        if (e.data && e.data.type === 'bigpicture-osk-input' && lastFocusedInput) {
+          lastFocusedInput.value = e.data.value;
+          lastFocusedInput.dispatchEvent(new Event('input', { bubbles: true }));
+          lastFocusedInput.dispatchEvent(new Event('change', { bubbles: true }));
+        } else if (e.data && e.data.type === 'bigpicture-osk-submit' && lastFocusedInput) {
+          // Submit the form or trigger search
+          const form = lastFocusedInput.closest('form');
+          if (form) {
+            form.dispatchEvent(new Event('submit', { bubbles: true, cancelable: true }));
+            // Also try clicking any submit button
+            const submitBtn = form.querySelector('button[type="submit"], input[type="submit"], button:not([type])');
+            if (submitBtn) submitBtn.click();
+          }
+          // Trigger Enter key event
+          lastFocusedInput.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', keyCode: 13, bubbles: true }));
+          lastFocusedInput.dispatchEvent(new KeyboardEvent('keypress', { key: 'Enter', keyCode: 13, bubbles: true }));
+          lastFocusedInput.dispatchEvent(new KeyboardEvent('keyup', { key: 'Enter', keyCode: 13, bubbles: true }));
+        }
+      });
+      
+      console.log('[BigPicture] Input focus detection injected');
+    })();
+  `;
+  
+  webview.executeJavaScript(script).catch(err => {
+    console.log('[BigPicture] Could not inject input detection:', err);
+  });
+}
+
 function exitBigPictureMode() {
   console.log('[BigPicture] Exiting Big Picture Mode');
   
@@ -1334,8 +1564,8 @@ function virtualClick(rightClick = false) {
   const containerRect = container.getBoundingClientRect();
   
   // Calculate position relative to webview
-  const x = state.cursorX - containerRect.left;
+  const x = Math.round(state.cursorX - containerRect.left);
-  const y = state.cursorY - containerRect.top;
+  const y = Math.round(state.cursorY - containerRect.top);
   
   // Show click animation
   if (state.cursorElement) {
@@ -1343,56 +1573,191 @@ function virtualClick(rightClick = false) {
     setTimeout(() => state.cursorElement.classList.remove('clicking'), 150);
   }
   
-  // Send mouse event to webview
+  const webview = state.currentWebview;
-  try {
-    const webContents = state.currentWebview;
   
-    // Use executeJavaScript to simulate click at coordinates
+  // Try to use native input event injection via IPC (most reliable for complex sites)
-    const clickScript = rightClick ? `
+  if (state.webviewContentsId && window.bigPictureAPI && window.bigPictureAPI.sendInputEvent) {
-      (function() {
+    const sendNativeClick = async () => {
-        const el = document.elementFromPoint(${x}, ${y});
+      try {
-        if (el) {
+        // Send mouseMove first to position the cursor
-          const event = new MouseEvent('contextmenu', {
+        await window.bigPictureAPI.sendInputEvent(state.webviewContentsId, {
-            bubbles: true,
+          type: 'mouseMove',
-            cancelable: true,
+          x: x,
-            clientX: ${x},
+          y: y
-            clientY: ${y},
+        });
-            button: 2
+        
-          });
+        // Small delay then send mouseDown
-          el.dispatchEvent(event);
+        await new Promise(r => setTimeout(r, 10));
-        }
+        
-      })();
+        await window.bigPictureAPI.sendInputEvent(state.webviewContentsId, {
-    ` : `
+          type: 'mouseDown',
-      (function() {
+          x: x,
-        const el = document.elementFromPoint(${x}, ${y});
+          y: y,
-        if (el) {
+          button: rightClick ? 'right' : 'left',
-          // Try to focus if it's an input
+          clickCount: 1
-          if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.contentEditable === 'true') {
+        });
-            el.focus();
+        
-          }
+        // Small delay then send mouseUp
-          // Simulate full click sequence
+        await new Promise(r => setTimeout(r, 50));
-          const rect = el.getBoundingClientRect();
+        
-          const events = ['mousedown', 'mouseup', 'click'];
+        await window.bigPictureAPI.sendInputEvent(state.webviewContentsId, {
-          events.forEach(type => {
+          type: 'mouseUp',
-            const event = new MouseEvent(type, {
+          x: x,
+          y: y,
+          button: rightClick ? 'right' : 'left',
+          clickCount: 1
+        });
+        
+        console.log('[BigPicture] Native click sent at', x, y);
+      } catch (err) {
+        console.log('[BigPicture] Native input error, falling back to JS:', err);
+        fallbackJavaScriptClick(webview, x, y, rightClick);
+      }
+    };
+    
+    sendNativeClick();
+    return;
+  }
+  
+  // Fallback to JavaScript injection
+  fallbackJavaScriptClick(webview, x, y, rightClick);
+}
+
+function fallbackJavaScriptClick(webview, x, y, rightClick) {
+  try {
+    if (rightClick) {
+      // For right-click, use JavaScript injection
+      const rightClickScript = `
+        (function() {
+          const el = document.elementFromPoint(${x}, ${y});
+          if (el) {
+            const event = new MouseEvent('contextmenu', {
               bubbles: true,
               cancelable: true,
-              view: window,
               clientX: ${x},
               clientY: ${y},
-              button: 0
+              button: 2
             });
             el.dispatchEvent(event);
-          });
+          }
-          // Also try clicking directly for links and buttons
+        })();
-          if (el.click) el.click();
+      `;
-        }
+      webview.executeJavaScript(rightClickScript).catch(err => {
-      })();
+        console.log('[BigPicture] Right-click injection error:', err);
-    `;
+      });
+    } else {
+      // Comprehensive JavaScript injection with pointer events
+      const clickScript = `
+        (function() {
+          const x = ${x};
+          const y = ${y};
+          const el = document.elementFromPoint(x, y);
+          if (!el) return;
           
-    webContents.executeJavaScript(clickScript).catch(err => {
+          // Check if we're clicking on YouTube player area
-      console.log('[BigPicture] Click injection error:', err);
+          const isYouTubePlayer = el.closest('.html5-video-player') || 
-    });
+                                  el.closest('.ytp-player') ||
+                                  el.closest('#movie_player') ||
+                                  el.closest('.html5-main-video') ||
+                                  el.closest('.video-stream') ||
+                                  (window.location.hostname.includes('youtube.com') && 
+                                   (el.tagName === 'VIDEO' || el.closest('#player')));
+          
+          if (isYouTubePlayer) {
+            // For YouTube player, directly toggle playback
+            const video = document.querySelector('video.html5-main-video') || 
+                         document.querySelector('video.video-stream') ||
+                         document.querySelector('#movie_player video') ||
+                         document.querySelector('video');
+            if (video) {
+              if (video.paused) {
+                video.play().catch(() => {});
+              } else {
+                video.pause();
+              }
+              return;
+            }
+          }
+          
+          // Find the actual clickable element (may be parent)
+          let clickTarget = el;
+          let current = el;
+          for (let i = 0; i < 10 && current; i++) {
+            if (current.tagName === 'A' || current.tagName === 'BUTTON' || 
+                current.onclick || current.getAttribute('role') === 'button' ||
+                window.getComputedStyle(current).cursor === 'pointer') {
+              clickTarget = current;
+              break;
+            }
+            current = current.parentElement;
+          }
+          
+          // Common event options
+          const eventOptions = {
+            bubbles: true,
+            cancelable: true,
+            view: window,
+            clientX: x,
+            clientY: y,
+            screenX: x,
+            screenY: y,
+            button: 0,
+            buttons: 1,
+            pointerId: 1,
+            pointerType: 'mouse',
+            isPrimary: true,
+            pressure: 0.5,
+            width: 1,
+            height: 1
+          };
+          
+          // Handle input elements specially - focus first
+          const isInput = el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || 
+                         el.contentEditable === 'true' || el.isContentEditable ||
+                         el.getAttribute('role') === 'textbox' || el.getAttribute('role') === 'searchbox' ||
+                         el.closest('[contenteditable="true"]');
+          
+          if (isInput) {
+            // Focus the input element
+            el.focus();
+            // Dispatch proper focus sequence
+            el.dispatchEvent(new FocusEvent('focus', { bubbles: true }));
+            el.dispatchEvent(new FocusEvent('focusin', { bubbles: true }));
+            // Dispatch click to activate any click handlers
+            el.dispatchEvent(new MouseEvent('click', eventOptions));
+            return;
+          }
+          
+          // For general video elements (not YouTube specific)
+          if (el.tagName === 'VIDEO') {
+            if (el.paused) {
+              el.play().catch(() => {});
+            } else {
+              el.pause();
+            }
+            return;
+          }
+          
+          // Dispatch pointer events (used by modern sites)
+          try {
+            clickTarget.dispatchEvent(new PointerEvent('pointerdown', eventOptions));
+            clickTarget.dispatchEvent(new PointerEvent('pointerup', eventOptions));
+          } catch(e) {}
+          
+          // Dispatch mouse events
+          clickTarget.dispatchEvent(new MouseEvent('mousedown', eventOptions));
+          clickTarget.dispatchEvent(new MouseEvent('mouseup', eventOptions));
+          clickTarget.dispatchEvent(new MouseEvent('click', eventOptions));
+          
+          // Direct click as final fallback
+          if (clickTarget.click) clickTarget.click();
+        })();
+      `;
+      
+      webview.executeJavaScript(clickScript).catch(err => {
+        console.log('[BigPicture] Click injection error:', err);
+      });
+    }
   } catch (err) {
     console.log('[BigPicture] Virtual click error:', err);
   }