To reproduce:

        Me.RadDateTimePicker1.Format = DateTimePickerFormat.Custom
        Me.RadDateTimePicker1.CustomFormat = "MM/dd/yyyy"

        Dim provider As MaskDateTimeProvider = TryCast(Me.RadDateTimePicker1.DateTimePickerElement.TextBoxElement.Provider, MaskDateTimeProvider)
        provider.AutoSelectNextPart = True

 If you type in 01/01/50, RadDateTimePicker displays 01/01/1950 which is correct. If you type in 01/01/2050, the system is again changing the century and display 01/01/1950 which is incorrect. 

Workaround: use a custom mask provider to customize how the autocompletion work:
    Public Class CustomMaskDateTimeProvider
        Inherits Telerik.WinControls.UI.MaskDateTimeProvider
        Public Sub New(mask As String, culture As CultureInfo, owner As RadMaskedEditBoxElement)
            MyBase.New(mask, culture, owner)

        End Sub

        Protected Overrides Function HandleKeyPress(part As MaskPart, e As KeyPressEventArgs) As Boolean
            Dim value As Integer = 0
            If Not Integer.TryParse(e.KeyChar.ToString(), value) Then
                Return True
            End If

            Dim stringValue As String = String.Empty
            If part.type = PartTypes.MiliSeconds Then
                Dim newValue As Integer = CInt((part.value / Math.Pow(10, (3 - part.len)))) * 10 + value
                part.value = newValue Mod CInt(Math.Pow(10, part.len))
            Else
                stringValue = part.value.ToString()
                If stringValue.Length = part.len AndAlso stringValue.Length >= part.maskPart.Length Then
                    stringValue = stringValue.Substring(1)
                End If

                If Me.AutoCompleteYear AndAlso stringValue.Length = 1 Then
                    stringValue = String.Empty
                End If

                part.value = Integer.Parse(stringValue & e.KeyChar)
            End If

            If part.type = PartTypes.Year AndAlso part.maskPart.Length = 2 Then
                Dim len As Integer = part.value.ToString().Length
                part.value = Integer.Parse(String.Format("{0}{1}", DateTime.Now.Year.ToString().Substring(0, 2), part.value.ToString().Substring(len - 2, 2)))
            End If

            If Me.AutoCompleteYear AndAlso part.type = PartTypes.Year Then
                Dim len As Integer = part.value.ToString().Length
                If len >= 2 Then
                    part.value = Integer.Parse(part.value.ToString().Substring(len - 2, 2))
                End If

                'If part.value >= 50 AndAlso part.value <= 99 Then
                '    part.value += 1900
                'ElseIf part.value >= 500 AndAlso part.value <= 999 Then
                '    part.value += 1000
                'ElseIf part.value < 50 OrElse (part.value > 99 AndAlso part.value < 500) Then
                part.value += 2000
                ' End If
            End If

            If part.value > part.max OrElse part.value < part.min Then
                part.value = value
            End If

            Return True
        End Function
         
    End Class

  Me.RadDateTimePicker1.Format = DateTimePickerFormat.Custom
        Me.RadDateTimePicker1.CustomFormat = "MM/dd/yyyy"

        Me.RadDateTimePicker1.DateTimePickerElement.TextBoxElement.Provider = New CustomMaskDateTimeProvider(Me.RadDateTimePicker1.DateTimePickerElement.TextBoxElement.Mask, Me.RadDateTimePicker1.Culture, Me.RadDateTimePicker1.DateTimePickerElement.TextBoxElement)

        Dim provider As MaskDateTimeProvider = TryCast(Me.RadDateTimePicker1.DateTimePickerElement.TextBoxElement.Provider, MaskDateTimeProvider)
        provider.MaxDate = Me.RadDateTimePicker1.DateTimePickerElement.MaxDate
        provider.AutoSelectNextPart = True
        provider.AutoCompleteYear = True

Thus, the year always be 2000+.