1. Add a RadMaskedEditBox and a RadCheckbox to your form.
2. Add ToggleStateChanged event to the RadCheckbox.
3. Use the following code:
private void radCheckBox1_ToggleStateChanged(object sender, StateChangedEventArgs args)
{
    if (args.ToggleState == Telerik.WinControls.Enumerations.ToggleState.On)
    {
        radMaskedEditBox1.Mask = "^[0-9]{5}$";
        radMaskedEditBox1.MaskType = MaskType.Regex;
    }
 
    else if (args.ToggleState == Telerik.WinControls.Enumerations.ToggleState.Off)
    {
        radMaskedEditBox1.Mask = "";
        radMaskedEditBox1.MaskType = MaskType.None;
    }
}
4. Start the application
5. Check and uncheck the checkbox (now ToggleStateChanged event was fired)
6. Type some letters to the RadMaskedEditBox (e.g. "aaa")
7. Press tab

Workaround:
RegexMaskTextBoxProvider provider;
private void radCheckBox1_ToggleStateChanged(object sender, Telerik.WinControls.UI.StateChangedEventArgs args)
{
    if (args.ToggleState == Telerik.WinControls.Enumerations.ToggleState.On)
    {
        radMaskedEditBox1 = new RadMaskedEditBox();
        radMaskedEditBox1.Mask = "^[0-9]{5}$";
        radMaskedEditBox1.MaskType = MaskType.Regex;
        radMaskedEditBox1.MaskedEditBoxElement.TextBoxItem.HostedControl.Leave += HostedControl_Leave;
        provider = (RegexMaskTextBoxProvider) radMaskedEditBox1.MaskedEditBoxElement.Provider;
    }

    else if (args.ToggleState == Telerik.WinControls.Enumerations.ToggleState.Off)
    {
        radMaskedEditBox1.MaskedEditBoxElement.Mask = "";
        radMaskedEditBox1.MaskType = MaskType.None;
    }
}

private void HostedControl_Leave(object sender, EventArgs e)
{
    if (radMaskedEditBox1.MaskType == MaskType.None && provider != null)
    {
        provider.GetType().GetField("mask", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance).SetValue(provider, "*");
    }
}