Custom Keyboard: handling inputType change

I am running into an issue that I can't figure out. I wrote a simple custom IME keyboard based on this sample.

It basically has two custom keyboards, one for letters and one for numbers. They use different layouts.

However, when I add two EditText controls one for text and one for numbers, the keyboard does not refresh to the type it belongs. What I mean is that if I select the EditText with inputType="text" first, the QWERTY keyboard layout comes up. But then when I select the second EditText with inputType="number" the QWERTY keyboard shows up again. However it is supposed to load a different layout for numbers that is wired into the code.

In other words, here's the test activity layout:

Now if I select the "Text" field, the QWERTY keyboard comes up as below:

However, if I select the "Number" filed, the QWERTY keyboard still shows up which is wrong.

The expected behavior would be this keyboard to show up.

Here's the code for the CustomIME and I tried to use postInvalidate() on the view, pre-load all layouts during onInitializeInterface() but nothing worked. It never switches to the number's layout properly

public class CustomIME extends InputMethodService
        implements KeyboardView.OnKeyboardActionListener {

    public static final String CUSTOM_IME = "CUSTOM_IME";
    private KeyboardView mKeyboardView;
    private Keyboard mKeyboardCurrent;
    private KeyboardType mKeyboardTypeCurrent = KeyboardType.QWERTY_LETTERS;
    private boolean mCAPs = false;


    enum KeyboardType {
        QWERTY_LETTERS,
        NUMBERS
    }

    @Override
    public View onCreateInputView() {
        loadCurrentKeyboard();
        mKeyboardView = (KeyboardView) getLayoutInflater().inflate(R.layout.custom_ime_keyboard, null);
        mKeyboardView.setBackgroundResource(R.drawable.btn_gradient);
        mKeyboardView.setOnKeyboardActionListener(this);

        if (mKeyboardCurrent != null) {
            mKeyboardView.setKeyboard(mKeyboardCurrent);
        }

        return mKeyboardView;
    }

    @Override
    public void onInitializeInterface() {
        // tried loading everything here but did not make a difference
    }

    private void loadCurrentKeyboard() {
        if (mKeyboardTypeCurrent == KeyboardType.QWERTY_LETTERS) {
            mKeyboardCurrent = new Keyboard(getApplicationContext(), R.xml.custom_ime_qwerty);
        } else if (mKeyboardTypeCurrent == KeyboardType.NUMBERS) {
            mKeyboardCurrent = new Keyboard(getApplicationContext(), R.xml.custom_ime_number);
        } else {
            Log.e(CUSTOM_IME, "Invalid keyboard type");
        }
    }

    @Override
    public void onStartInput(EditorInfo attribute, boolean restarting) {
        super.onStartInput(attribute, restarting);
        switch (attribute.inputType & InputType.TYPE_MASK_CLASS) {
            case InputType.TYPE_CLASS_NUMBER:
                boolean signed = (attribute.inputType & InputType.TYPE_NUMBER_FLAG_SIGNED) != 0;
                boolean decimal = (attribute.inputType & InputType.TYPE_NUMBER_FLAG_DECIMAL) != 0;

                // set default
                mKeyboardTypeCurrent = KeyboardType.QWERTY_LETTERS;
                if (!signed && !decimal) {
                    mKeyboardTypeCurrent = KeyboardType.NUMBERS;
                }
                break;
            case InputType.TYPE_CLASS_TEXT:
            default:
                mKeyboardTypeCurrent = KeyboardType.QWERTY_LETTERS;
        }

        // This did not make a difference
        if (mKeyboardView != null) {
            mKeyboardView.postInvalidate();
        }
    }

    @Override
    public void onKey(int primaryCode, int[] keyCodes) {
        InputConnection inputConnection = getCurrentInputConnection();
        switch (primaryCode) {
            default:
                char asciiCode = (char) primaryCode;
                if (Character.isLetter(asciiCode) && mCAPs) {
                    asciiCode = Character.toUpperCase(asciiCode);
                }
                inputConnection.commitText(String.valueOf(asciiCode), 1);
        }
    }
}

And the layouts:

custom_ime_keyboard.xml:

<?xml version="1.0" encoding="UTF-8"?>
<android.inputmethodservice.KeyboardView xmlns:android="http://schemas.android.com/apk/res/android"
    android:id="@+id/custom_ime_keyboard_id1"
    android:layout_width="match_parent"
    android:layout_height="wrap_content"
    android:layout_alignParentBottom="true"
    android:keyPreviewLayout="@layout/custom_ime_preview" />

activity_main.xml

<LinearLayout
    android:id="@+id/layout1"
    android:layout_width="wrap_content"
    android:layout_height="wrap_content"
    android:layout_alignParentTop="true"
    android:layout_centerHorizontal="true"
    android:layout_margin="10dp"
    android:orientation="horizontal">

    <EditText
        android:id="@+id/edit1"
        android:layout_width="100dp"
        android:layout_height="60dp"
        android:inputType="text"
        android:hint="Text"
        android:padding="10dp"
        android:textSize="12sp" />

    <EditText
        android:id="@+id/edit2"
        android:layout_width="100dp"
        android:layout_height="60dp"
        android:hint="Number"
        android:inputType="number"
        android:padding="10dp"
        android:textSize="12sp" />
</LinearLayout>

Finally the keyboard layouts (custom_ime_qwerty.xml, and custom_ime_number.xml).

<Keyboard xmlns:android="http://schemas.android.com/apk/res/android"
    android:keyHeight="64dp"
    android:keyWidth="9%p">
    <!--1st row-->
    <Row android:rowEdgeFlags="top">
        <Key
            android:codes="113"
            android:keyEdgeFlags="left"
            android:keyLabel="q" />
        <Key
            android:codes="119"
            android:keyLabel="w" />
        <Key
            android:codes="101"
            android:keyLabel="e" />
        <Key
            android:codes="114"
            android:keyLabel="r" />
        etc...


<Keyboard xmlns:android="http://schemas.android.com/apk/res/android"
    android:keyHeight="64dp"
    android:keyWidth="20%p"
    android:label="number"
    android:verticalGap="0px">
    <!--1st row-->
    <Row android:rowEdgeFlags="top">
        <Key
            android:codes="49"
            android:keyEdgeFlags="left"
            android:keyLabel="1" />
        <Key
            android:codes="50"
            android:keyLabel="2" />
        <Key
            android:codes="51"
            android:keyLabel="3" />