How to reproduce: 
Public Class Form1
    Sub New()

        InitializeComponent()

        Dim col As New GridViewTextBoxColumn("Column1")
        Me.RadGridView1.Columns.Add(col)
    End Sub

    Private Sub RadButton1_Click(sender As Object, e As EventArgs) Handles RadButton1.Click
        Dim sw As New Stopwatch
        sw.Start()

        Me.RadGridView1.Rows.Clear()
        Me.RadGridView1.BeginUpdate()
        Dim RowInfo
        For ii = 1 To 100000
            RowInfo = Me.RadGridView1.Rows.AddNew
            RowInfo.Cells("Column1").Value = ii
        Next
        Me.RadGridView1.EndUpdate()

        sw.Stop()
        Console.WriteLine("Elapsed: " & sw.Elapsed.TotalSeconds)
    End Sub
End Class

Workaround: add data to a collection and use bound mode