How to reproduce: check the attached project and video

Workaround:
public partial class RadForm1 : Telerik.WinControls.UI.RadForm
{
    public RadForm1()
    {
        InitializeComponent();
        radDock1.FloatingWindowCreated += RadDock1_FloatingWindowCreated;
    }

    private void RadDock1_FloatingWindowCreated(object sender, Telerik.WinControls.UI.Docking.FloatingWindowEventArgs e)
    {
        e.Window = new MyWindow(this.radDock1);
    }
}

public class MyWindow : FloatingWindow
{
    SizeF oldDpi = new SizeF(1, 1);

    public MyWindow(RadDock dockManager) : base (dockManager)
    { }
    protected override void HandleDpiChanged()
    {
        
        base.HandleDpiChanged();
        SizeF descale = new SizeF(1f / this.FormElement.DpiScaleFactor.Width, 1f / this.FormElement.DpiScaleFactor.Height);
        this.Scale(descale);
        var dpi = NativeMethods.GetMonitorDpi(Screen.FromRectangle(this.Bounds), NativeMethods.DpiType.Effective);
        if (oldDpi != dpi)
        {
            SizeF sz = new SizeF(dpi.Width / oldDpi.Width, dpi.Height / oldDpi.Height);

            this.Scale(dpi);
        }

        oldDpi = dpi;
    }
}