To reproduce:

BindingList<Item> items = new BindingList<Item>();

public Form1()
{
    InitializeComponent();

    Item dataItem1 = new Item(1);
    dataItem1.Name = "Cat";
    dataItem1.GroupName = "Animal";
    dataItem1.GroupSortPriority = 2;
    items.Add(dataItem1);
    Item dataItem2 = new Item(2);
    dataItem2.Name = "Kitten";
    dataItem2.GroupName = "Animal";
    dataItem2.GroupSortPriority = 2;
    dataItem2.ParentIndex = 1;
    items.Add(dataItem2);
    Item dataItem3 = new Item(3);
    dataItem3.Name = "Trout";
    dataItem3.GroupName = "Fish";
    dataItem3.GroupSortPriority = 1;
    items.Add(dataItem3);
    radGridView1.Relations.AddSelfReference(radGridView1.MasterTemplate, "Index", "ParentIndex");
    radGridView1.DataSource = items;
}

public class Item
{
    public Item(int index)
    {
        Index = index;
    }

    public int Index { get; private set; }

    public int ParentIndex { get; set; }

    public string Name { get; set; }

    public string GroupName { get; set; }

    public int GroupSortPriority { get; set; }
}

public class GridGroupComparer : IComparer<Group<GridViewRowInfo>>
{
    public GridGroupComparer()
    {
    }

    public int Compare(
        Group<GridViewRowInfo> x,
        Group<GridViewRowInfo> y)
    {
        if (null == x.Key || null == y.Key)
            return 1;

        GridViewRowInfo xGridViewRowInfo = x.DefaultIfEmpty(null).First();
        GridViewRowInfo yGridViewRowInfo = y.DefaultIfEmpty(null).First();

        if (null == xGridViewRowInfo || null == yGridViewRowInfo)
            return x.Key.GetHashCode().CompareTo(y.Key.GetHashCode());

        Item xGridRowDataItem = xGridViewRowInfo.DataBoundItem as Item;
        Item yGridRowDataItem = yGridViewRowInfo.DataBoundItem as Item;

        if (null == xGridRowDataItem || null == yGridRowDataItem)
            return x.Key.GetHashCode().CompareTo(y.Key.GetHashCode());

        if (xGridRowDataItem.GroupSortPriority > yGridRowDataItem.GroupSortPriority)
            return 1;
        if (xGridRowDataItem.GroupSortPriority < yGridRowDataItem.GroupSortPriority)
            return -1; 

        return 0;
    }
}

private void Form1_Load(object sender, EventArgs e)
{ 
    GridGroupComparer groupComparer = new GridGroupComparer();
    radGridView1.MasterTemplate.GroupComparer = groupComparer;

    GroupDescriptor descriptor = new GroupDescriptor();
    descriptor.GroupNames.Add("GroupName", ListSortDirection.Ascending);
    radGridView1.GroupDescriptors.Add(descriptor);
}

If you remove the Self-referencing hierarchy, you will notice that the GroupComparer is respected.