Virtual Keyboard/IME API

Introduction

Virtual Keyboard/IME API aims to implement the system IME as a Web App. It will be used in Firefox OS and use cases could be found in the Firefox OS Keyboard UX document(WIP).

The API provides the communication channel between the IME App and the other App that receives user's inputs.

It is very different from the IME API from Google that aims to re-use the system's IME in a web page.

Status

API discussion:

Implementation:

bug 737110 - Bug 737110 - Virtual Keyboard API
bug 805586 - [keyboard] keyboard needs a 'hide keyboard' button(main tracking bug)
bug 844716 - Enable keyboard Apps to get/set selection range of the input field
bug 860546 - [keyboard] JS changes to a textfield while keyboard is displayed do not get passed to keyboard
bug 861665 - Allow IME to get notification when text field content is changed
bug 861515 - Keyboard should be able to modify the text of the input field directly
bug 838308 - mozKeyboard should require a permission to use
bug 842436 - Keyboard API: There should only be one keyboard active, and Gecko should block interaction from non-active keyboards

Features

The Virtual Keyboard/IME API supports the following features:

Notifies the VKB app when the focus text field was changing in other

apps

Allow user to manual hide the keyboard. Check bug 737110.
The VKB app should be responsive to properties and the state of the input field (more than HTML5 input type, including current content, cursor position, x-inputmode bug 796544).
Sends trust
The VKB app should be able to send trusted key events such as they are considered by the other apps as user' inputs.
The VKB app should be able to send a character or a string to the current input cursor position.
Keyboard should be able to overwrite the current value of the input field of the input field and set the cursor position.
The VKB app should be able to move the cursor position or select a specified range of text.
The VKB should be able to switch the focus onto the previous/next input field.
The return key label of the VKB can be customized.

Proposed API

partial interface Navigator {
    readonly attribute InputMethodManager mozInputMethodManager;
    readonly attribute InputMethodConnection mozInputMethodConnection;
};

dictionary InputMethodInfo {
    url: 'XXX',
    name: 'IME name',
    locale: 'en-US',
};

// Manages the list of IMEs, enables/disables IME and switches to an IME.
interface InputMethodManager {
    // Show a (system) menu for all enabled input methods that allow user to select.
    void showInputMethodPicker();

    // Get a list of all installed IMEs.
    Promise<InputMethodInfo[]> getAll();

    // Get a list of all enabled IMEs.
    Promise<InputMethodInfo[]> getEnabled();

    // Get input method info of the caller IME.
    Promise<InputMethodInfo> getSelf();

    // Switch to the given IME.
    // Privileged API. Only the system app and current IME can switch IME.
    Promise<boolean> setCurrentInputMethod(InputMethodInfo info);

    // Switch to next IME.
    // We may not need this method and use setCurrentInputMethod instead.
    Promise<boolean> switchToNextInputMethod();

    // Get current IME.
    Promise<InputMethodInfo> getCurrentInputMethod();

    // Enable an IME.
    // Privileged API. Only the system app can eanble an IME.
    Promise<boolean> enable(InputMethodInfo info);

    // Disable an IME.
    // Privileged API. Only the system app can disable an IME.
    Promise<boolean> disable(InputMethodInfo info);
 };

// The input context, which are attributes and information of current input field.
dictionary InputMethodContext {
   // This is used to specify the target of input field operations. This ID becomes invalid as soon as user leaves input field and blur event is sent.
   long contextId;

   // The tag name of input field, which is enum of "input", "textarea", or "contenteditable"
   DOMString name;
   // The type of the input field, which is enum of text, number, password, url, search and email.
   DOMString type;
    /*
     * The input mode string.
     * https://bugzilla.mozilla.org/show_bug.cgi?id=796544
     * It can be one of the following values:
     * "none"
     * "verbatim"    - no capitalization, no word suggestions
     * "latin"       - word suggestions but no capitalization
     * "latin-prose" - word suggestions and capitalization at the start of sentences
     * "latin-name"  - word suggestions and capitalize each word
     * "digit"      - digits(0-9) only.
     */
   DOMString inputmode;
   /*
    * The primary language for the input field.
    * It is the value of HTMLElement.lang.
    * see http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#htmlelement
    */
    DOMString lang;
};

interface InputMethodConnection: EventTarget {
   // It informs the IME that text input has started in an input field. If the IME is activated, this event is sent when focus enters a new input field. Otherwise this event is sent when user switches to the IME and activates it.
   attribute EventHandler onstart;

   // It informs the IME that text input has finished in last input field. This event is sent when focus leaves current input field or user switches to another IME.
   attribute EventHandler onfinish;

   // This event is sent when the attributes of input context has changed, such as type, but focus has not.
   attribute EventHandler oncontextchange;

   /*
    * This event is sent when the text around the cursor is changed, due to either text
    * editing or cursor movement. The text length is limited to 100 characters for each
    * back and forth direction.
    *
    * The event handler function is specified as:
    * @param beforeString Text before and including cursor position.
    * @param afterString Text after and excluing cursor position.
    * function(long contextId, DOMString beforeText, DOMString afterText) {
    * ...
    *  }
    */
    attribute SurroundingTextChangeEventHandler onsurroundingtextchange;

   // User moves the cursor, changes the selection, or alters the composing text length
   attribute EventHandler onselectionchange;

   // TODO: maybe the parameters could be simpler?
    Promise<boolean> sendKey(long contextId, long keyCode, long charCode, long modifiers);
   // Or Promise<boolean> sendKey(long contextId, KeyboardEvent event)

   /*
    *  Get the whole text content of the input field.
    */
    Promise<DOMString> getText(long contextId);

   /*
    * Commit text to current input field and replace text around cursor position. It will clear the current composition.
    *
    * @param text The string to be replaced with.
    * @param offset The offset from the cursor position where replacing starts. Defaults to 0.
    * @param length The length of text to replace. Defaults to 0.
    */
    Promise<boolean> commitText(long contextId, DOMString text, [optional] long offset, [optional] long length);

    /*
     *
     * Delete text around the cursor.
     * @param offset The offset from the cursor position where deletion starts.
     * @param length The length of text to delete.
     */
    Promise<boolean> deleteSurroundingText(long offset, long length);

    // The start and stop position of the selection.
    readonly attribute long selectionStart;
    readonly attribute long selectionEnd;

    /*
     * Set the selection range of the the editable text.
     * Note: This method cannot be used to move the cursor during composition. Calling this
     * method will cancel composition.
     * @param start The beginning of the selected text.
     * @param length The length of the selected text.
     *
     * Note that the start position should be less or equal to the end position.
     * To move the cursor, set the start and end position to the same value.
     */
    Promise<boolean> setSelectionRange(long contextId, long start, long length);

    /*
     * Set current composition. It will start or update composition.
     * @param cursor Position in the text of the cursor.
     */
    Promise<boolean> setComposition(long contextId, DOMString text, long cursor);

    /*
     * Clear composition. Called to cancel composition.
     * Use commitText to end composition and commit text.
     */
    Promise<boolean> clearComposition(long contextId);

    // Clear the focus of the current input field and hide the keyboard.
    Promise<boolean> removeFocus(long contextId);

    // The input method context
    readonly attribute InputMethodContext elementInfo;
};

[TreatNonCallableAsNull]
callback SurroundingTextChangeEventHandlerNonNull = void (long contextId, DOMString beforeText, DOMString afterText);
typedef SurroundingTextChangeEventHandlerNonNull? SurroundingTextChangeEventHandler;

Examples

var conn = navigator.mozInputMethodConnection;

// Called when the user starts or finishes editing an input field
conn.oninputmethodstatechange= function(event) {
    if (conn.inputStarted) {
        // Text input should start
    } else {
        // Text input should end
    }
};

// Insert a string at the current cursor position
conn.replaceSurroundingText('Hello world');

// Clear delete 5 characters before the cursor position.
conn.replaceSurroundingText(, 5);

// Get the selected text
conn.getText(conn.selectionStart, conn.selectionEnd, function callback(text) {
    var selectedText = text;
});

// Get notified when the text content has changed.
conn.ontextchange = function() {
  conn.getText(0, conn.textLength, function(text) {
    console.log(text);
  }
}

// Move the cursor position
var position = 10;
conn.setSelectionRange(position, position);

// Hide the keyboard
conn.removeFocus();

// Switch the focus onto the next input field if possible
if (conn.canAdvanceFocus) {
    conn.advanceFocus();
}

WebAPI/KeboardIME