Use attached to reproduce.
Case 1 - the initial scaling is not correct and the showing dilog1 results in a different DPI each time. 
Case 2 - the AllowTheming property disabled the scaling - see dialog 2.

Workaround:

this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Dpi;
this.AutoScaleDimensions = new System.Drawing.SizeF(96F, 96F);

public class MyRadForm : RadForm
{
    protected override void HandleDpiChanged()
    {
        if (TelerikHelper.IsWindows10CreatorsUpdateOrHigher)
        {
            return;
        }

        base.HandleDpiChanged();
    }

}