When clicking with the right mouse-button on one of the system buttons, and then dragging the mouse over the button until leaving it, without releasing the mouse-button itself, the button stays marked. It is reproducible in all themes.

Workaround:

public Form1()
{
    InitializeComponent();
 
    this.radRibbonBar1.RibbonBarElement.RibbonCaption.MinimizeButton.MouseLeave += Button_MouseLeave;
    this.radRibbonBar1.RibbonBarElement.RibbonCaption.MaximizeButton.MouseLeave += Button_MouseLeave;
    this.radRibbonBar1.RibbonBarElement.RibbonCaption.CloseButton.MouseLeave += Button_MouseLeave;
}

private void Button_MouseLeave(object sender, EventArgs e)
{
    RadImageButtonElement btn = sender as RadImageButtonElement;
    if (btn != null)
    {
        btn.IsMouseDown = false;
    }
}