The incorrect behavior is also observed if the SelectedObject is changed.

How to reproduce:

public partial class RadForm1 : RadForm
{
    public RadForm1()
    {
        this.InitializeComponent();
    }

    protected override void OnLoad(EventArgs e)
    {
        base.OnLoad(e);

        this.radPropertyGrid1.EnableSorting = true;
        this.radPropertyGrid1.SelectedObject = new TestObject();
        foreach (var item in this.radPropertyGrid1.Items)
        {
            if (item.Name == "A")
            {
                item.SortOrder = 2;
            }
            else if (item.Name == "B")
            {
                item.SortOrder = 1;
            }
            else
            {
                item.SortOrder = 0;
            }
        }

        this.radPropertyGrid1.PropertySort = PropertySort.NoSort;
    }
}

public class TestObject
{
    public int A { get; set; }
    public int C { get; set; }
    public int B { get; set; }
}

Workaround:
1. Use attributes in the model class:
public class TestObject
{
    [RadSortOrder(2)]
    public int A { get; set; }
    [RadSortOrder(0)]
    public int C { get; set; }
    [RadSortOrder(1)]
    public int B { get; set; }
}

2. Alternatively, toggle the PropertySort property in the Shown event of the form:
public partial class RadForm1 : RadForm
{
    public RadForm1()
    {
        this.InitializeComponent();
    }

    protected override void OnLoad(EventArgs e)
    {
        base.OnLoad(e);

        this.radPropertyGrid1.EnableSorting = true;
        this.radPropertyGrid1.SelectedObject = new TestObject();
        foreach (var item in this.radPropertyGrid1.Items)
        {
            if (item.Name == "A")
            {
                item.SortOrder = 2;
            }
            else if (item.Name == "B")
            {
                item.SortOrder = 1;
            }
            else
            {
                item.SortOrder = 0;
            }
        }

        this.radPropertyGrid1.PropertySort = PropertySort.NoSort;
    }

    protected override void OnShown(EventArgs e)
    {
        base.OnShown(e);

        this.radPropertyGrid1.PropertySort = PropertySort.Alphabetical;
        this.radPropertyGrid1.PropertySort = PropertySort.NoSort;
    }
}

public class TestObject
{
    public int A { get; set; }
    public int C { get; set; }
    public int B { get; set; }
}