To reproduce: please refer to the attached sample project and follow the steps illustrated in the gif file. 

Workaround: after loading the layout, set the MaximumSize for the FloatingWindow:
        public RadForm1()
        {
            InitializeComponent();

            this.radDock1.FloatingWindowCreated += radDock1_FloatingWindowCreated;
        }

        Size lastSize = Size.Empty;

        private void radDock1_FloatingWindowCreated(object sender, Telerik.WinControls.UI.Docking.FloatingWindowEventArgs e)
        {
          lastSize=  e.Window.Size;
        }

        private void button1_Click(object sender, EventArgs e)
        {
            int reps;
            if (!int.TryParse(radMaskedEditBox1.Text, out reps))
            {
                MessageBox.Show(this, "Enter a valid number for repititions.");
                return;
            }

            using (var dockLayout = new MemoryStream())
            {
                for (int i = 0; i < reps; i++)
                {
                    dockLayout.SetLength(0);
                    radDock1.SaveToXml(dockLayout);
                    radDock1.LoadFromXml(dockLayout);

                    foreach (Telerik.WinControls.UI.Docking.FloatingWindow fw in this.radDock1.ActiveFloatingWindows)
                    {
                        fw.MaximumSize = lastSize;
                    }
                }
            }
        }