To reproduce, use the following code:
((GridViewMaskBoxColumn)radGridView1.Columns["MaskEditBoxColumn"]).DataType = typeof(decimal);
((GridViewMaskBoxColumn)radGridView1.Columns["MaskEditBoxColumn"]).FormatString = "{0:(###) ###-####}";
((GridViewMaskBoxColumn)radGridView1.Columns["MaskEditBoxColumn"]).MaskType = MaskType.Standard;
((GridViewMaskBoxColumn)radGridView1.Columns["MaskEditBoxColumn"]).Mask = "(###) ###-####";
((GridViewMaskBoxColumn)radGridView1.Columns["MaskEditBoxColumn"]).TextMaskFormat = MaskFormat.ExcludePromptAndLiterals;

and add a numeric value in one of the column cells. Result -> Exception has been thrown by the target of an invocation.

Workaround: Use a custom type converter:
((GridViewMaskBoxColumn)radGridView1.Columns["MaskEditBoxColumn"]).DataTypeConverter = new MyConverter();



        public class MyConverter : TypeConverter
        {
            public override bool CanConvertFrom(ITypeDescriptorContext context, Type sourceType)
            {
                return sourceType == typeof(string);
            }

            public override bool CanConvertTo(ITypeDescriptorContext context, Type destinationType)
            {
                if (destinationType == typeof(string))
                {
                    return true;
                }
                return base.CanConvertTo(context, destinationType);
            }

            public override object ConvertTo(ITypeDescriptorContext context, CultureInfo culture, object value, Type destinationType)
            {
                if (destinationType == typeof(string))
                {
                    string s = value.ToString();
                    s= s.Insert(0, "(");

                    if (value.ToString().Length > 3)
                    {
                       s= s.Insert(4, ")");
                    }

                    if (value.ToString().Length > 7)
                    {
                        s= s.Insert(8, "-");
                    }

                    return s;
                }

                return base.ConvertTo(context, culture, value, destinationType);
            }

            public override object ConvertFrom(ITypeDescriptorContext context, CultureInfo culture, object value)
            {
                string s = value.ToString();
                s = s.Replace("(", "");
                s = s.Replace(")", "");
                s = s.Replace("-", "");
                return Convert.ToDecimal(s);

            }
        }