When an level 2 RadPanelBarItem is selected initially it is stretched. Pressing the Enter key will resize it so that its children fit. 

As a workaround you can subscribe for the PreviewKeyDown of RadPanelBar and handle the event if the item should not be collapsed.

private void OnPreviewKeyDown(object sender, KeyEventArgs e)
{
    var panelBarItem = sender as RadPanelBarItem;
    if (panelBarItem != null && panelBarItem.Level == 2 && e.Key == Key.Enter)
    {
        e.Handled = true;
    }
}