To reproduce:

List<Coordinate> coordinates_ = new List<Coordinate>();

public Form1()
{
    InitializeComponent();
   
    for (int i = 0; i < 10; i++)
    {
        coordinates_.Add(new Coordinate(i * 0.25, i * 0.33, i * 0.46));
    }

    this.radGridView1.AutoGenerateColumns = false;

    string mask = "F2";          

    this.radGridView1.Columns.Add(CreateDecimalColumn("X", "X", mask));
    this.radGridView1.Columns.Add(CreateDecimalColumn("Y", "Y", mask));
    this.radGridView1.Columns.Add(CreateDecimalColumn("Z", "Z", mask));
  
    this.radGridView1.AutoSizeColumnsMode = GridViewAutoSizeColumnsMode.Fill;

    SetRows();
}

GridViewDataColumn CreateDecimalColumn(string name, string headertext, string mask)
{
    var format = "{0:" + mask + "}";
    return new GridViewMaskBoxColumn()
    {
        Name = name,
        HeaderText = headertext,
        MinWidth = 50,
        MaskType = MaskType.Numeric,
        Mask = mask,
        FormatString = format,
        DataType = typeof(double),
        TextAlignment = ContentAlignment.MiddleRight
    };
}

void SetRows()
{
    foreach (var c in coordinates_)
    {
        var ri = radGridView1.Rows.AddNew();
        ri.Cells["X"].Value = c.X;
        ri.Cells["Y"].Value = c.Y;
        ri.Cells["Z"].Value = c.Z;
    }
}

public class Coordinate
{
    public double X { get; set; }

    public double Y { get; set; }

    public double Z { get; set; }

    public Coordinate(double x, double y, double z)
    {
        this.X = x;
        this.Y = y;
        this.Z = z;
    }
}

Workaround:
private void radGridView1_EditorRequired(object sender, EditorRequiredEventArgs e)
{
    if (e.EditorType==typeof(RadMaskedEditBoxEditor))
    {
        e.Editor = new RadMaskedEditBoxEditor();
    }
}