To reproduce: 
See attached video.

Workaround:
public Form1()
{
    RadDockEvents.TabStripItemCreating += RadDockEvents_TabStripItemCreating;
    InitializeComponent();
    radDock1.AutoHideAnimation = Telerik.WinControls.UI.Docking.AutoHideAnimateMode.None;

}
void RadDockEvents_TabStripItemCreating(object sender, TabStripItemCreatingEventArgs args)
{
    if (args.AutoHide)
    {
        var currentScale = args.TabItem.DpiScaleFactor;

        Screen showScreen = Screen.FromControl(this);
        SizeF scale = NativeMethods.GetMonitorDpi(showScreen, NativeMethods.DpiType.Effective);
        if (scale != currentScale)
        {
            var font = args.TabItem.Font;
            var newFont = new Font(font.Name, font.Size * scale.Width, font.Style);
            args.TabItem.Font = newFont;
        }
    }
}
protected override void OnClosed(EventArgs e)
{
    RadDockEvents.TabStripItemCreating -= RadDockEvents_TabStripItemCreating;
    base.OnClosed(e);
}