Workaround - hide the expander items if space is not enough:

 private void radGridView1_CellFormatting(object sender, CellFormattingEventArgs e)
 {
     GridDataCellElement cell = e.CellElement as GridDataCellElement;
     if (cell != null && cell.SelfReferenceLayout != null)
     {
         foreach (RadElement element in cell.SelfReferenceLayout.StackLayoutElement.Children)
         {
             GridExpanderItem expanderItem = element as GridExpanderItem;
             if (expanderItem != null)
             {
                 if (cell.ColumnInfo.Width < cell.SelfReferenceLayout.StackLayoutElement.Size.Width)
                 {
                     expanderItem.Opacity = 0;
                 }
                 else
                 {
                     expanderItem.Opacity = 1;
                 }
             }
         }
     }
 }