If you open a folder, but you doesn't select a child folder from the list shown in the explorer, when you click on the Open button the FileName property is empty.

To work this around you can create a custom RadOpenFolderDialog and override its GetViewModel. Then expose a public property with the view model. And when you close the dialog, check the IsDirectirySelected property of the view model. If so, return the Path property of the view model. Otherwise, use the standard approach and get the folder via the dialog's FileName property. 

public class CustomOpenFolderDialog : RadOpenFolderDialog
{
    public OpenFolderDialogViewModel ViewModel { get; internal set; }

    protected override OpenFolderDialogViewModel GetViewModel()
    {
        this.ViewModel = base.GetViewModel();            
        return this.ViewModel;
    }
}

---------------------------------

CustomOpenFolderDialog openFolderDialog = new CustomOpenFolderDialog();
openFolderDialog.Owner = this;
openFolderDialog.ShowDialog();
if (openFolderDialog.DialogResult == true)
{
     string folderName = String.Empty;
     if (!openFolderDialog.ViewModel.IsDirectorySelected)
     {
        folderName = openFolderDialog.ViewModel.Path;
     }
     else
     {
         folderName = openFolderDialog.FileName;
     }                
}