The close button in the document host is not higlighted (hovered-button-state-doc-host.png) as opposed to panes outside the DocumentHost area (hovered-button-state.png).

The workaround is to update the MouseOver visual state of the RadButtonDocumentCloseControlTemplate:

             <ControlTemplate x:Key="RadButtonDocumentCloseControlTemplate" TargetType="telerik:RadButton">
                <Grid x:Name="LayoutRoot" Background="Transparent" Width="18" Height="18">
                    <VisualStateManager.VisualStateGroups>
                        <VisualStateGroup x:Name="CommonStates">
                            <VisualState x:Name="Disabled">
                                <Storyboard>
                                    <DoubleAnimation Duration="0:0:0" To="0.3" Storyboard.TargetName="PathVisual" Storyboard.TargetProperty="Opacity"/>
                                </Storyboard>
                            </VisualState>
                            <VisualState x:Name="Normal"/>
                            <VisualState x:Name="MouseOver">
                                <Storyboard>
                                    <DoubleAnimation Duration="0:0:0" To="1" Storyboard.TargetName="MouseOverVisual" Storyboard.TargetProperty="Opacity"/>
                                </Storyboard>
                            </VisualState>
                            <VisualState x:Name="Pressed"/>
                        </VisualStateGroup>
                    </VisualStateManager.VisualStateGroups>
                    <Border x:Name="MouseOverVisual" Background="{telerik:Office2013Resource ResourceKey=EffectHighBrush}" Opacity="0"/>
                    <Path x:Name="PathVisual"
                    Width="8"
                    Height="8"
                    Data="M2.5,0 L4.5,0 4.5,2.5 7,2.5 7,4.5 4.5,4.5 4.5,7 2.5,7 2.5,4.5 0,4.5 0,2.5 2.5,2.5 z"
                    Stretch="Fill"
                    Fill="{TemplateBinding Background}"
                    RenderTransformOrigin="0.5,0.5"
                    SnapsToDevicePixels="True">
                        <Path.RenderTransform>
                            <RotateTransform Angle="-135"/>
                        </Path.RenderTransform>
                    </Path>
                </Grid>
            </ControlTemplate>
            <Style x:Key="CustomRadButtonDocumentCloseStyle" BasedOn="{StaticResource RadButtonDocumentCloseStyle}" TargetType="telerik:RadButton">
                <Setter Property="Template" Value="{StaticResource RadButtonDocumentCloseControlTemplate}"/>
            </Style>
            <ControlTemplate x:Key="RadPaneGroupDocumentHostControlTemplate" TargetType="telerik:RadPaneGroup">
                <Grid>
                    <VisualStateManager.VisualStateGroups>
                        <VisualStateGroup x:Name="CommonStates">
                            <VisualState x:Name="Disabled"/>
                            <VisualState x:Name="Normal"/>
                        </VisualStateGroup>
                        <VisualStateGroup x:Name="FocusStates">
                            <VisualState x:Name="Focused"/>
                            <VisualState x:Name="Unfocused"/>
                        </VisualStateGroup>
                        <VisualStateGroup x:Name="AutoCollapseStates">
                            <VisualState x:Name="SingleItem"/>
                            <VisualState x:Name="TwoOrMoreItems"/>
                        </VisualStateGroup>
                        <VisualStateGroup x:Name="PaneHeaderVisibilityStates">
                            <VisualState x:Name="PaneHeaderHidden"/>
                            <VisualState x:Name="PaneHeaderVisible"/>
                        </VisualStateGroup>
                        <VisualStateGroup x:Name="ResizerStates">
                            <VisualState x:Name="ResizerLeft">
                                <Storyboard>
                                    <ObjectAnimationUsingKeyFrames Storyboard.TargetName="DockResizer" Storyboard.TargetProperty="Visibility" Duration="0">
                                        <DiscreteObjectKeyFrame KeyTime="0">
                                            <DiscreteObjectKeyFrame.Value>
                                                <Visibility>Visible</Visibility>
                                            </DiscreteObjectKeyFrame.Value>
                                        </DiscreteObjectKeyFrame>
                                    </ObjectAnimationUsingKeyFrames>
                                    <ObjectAnimationUsingKeyFrames Storyboard.TargetName="DockResizer" Storyboard.TargetProperty="(Grid.RowSpan)" Duration="0">
                                        <DiscreteObjectKeyFrame KeyTime="0">
                                            <DiscreteObjectKeyFrame.Value>
                                                <sys:Int32>3</sys:Int32>
                                            </DiscreteObjectKeyFrame.Value>
                                        </DiscreteObjectKeyFrame>
                                    </ObjectAnimationUsingKeyFrames>
                                    <ObjectAnimationUsingKeyFrames Storyboard.TargetName="DockResizer" Storyboard.TargetProperty="Placement" Duration="0">
                                        <DiscreteObjectKeyFrame KeyTime="0">
                                            <DiscreteObjectKeyFrame.Value>
                                                <Dock>Left</Dock>
                                            </DiscreteObjectKeyFrame.Value>
                                        </DiscreteObjectKeyFrame>
                                    </ObjectAnimationUsingKeyFrames>
                                    <DoubleAnimation BeginTime="00:00:00" From="4" To="4" Storyboard.TargetName="DockResizer" Storyboard.TargetProperty="MinWidth"/>
                                </Storyboard>
                            </VisualState>
                            <VisualState x:Name="ResizerTop">
                                <Storyboard>
                                    <ObjectAnimationUsingKeyFrames Storyboard.TargetName="DockResizer" Storyboard.TargetProperty="Visibility" Duration="0">
                                        <DiscreteObjectKeyFrame KeyTime="0">
                                            <DiscreteObjectKeyFrame.Value>
                                                <Visibility>Visible</Visibility>
                                            </DiscreteObjectKeyFrame.Value>
                                        </DiscreteObjectKeyFrame>
                                    </ObjectAnimationUsingKeyFrames>
                                    <ObjectAnimationUsingKeyFrames Storyboard.TargetName="DockResizer" Storyboard.TargetProperty="(Grid.ColumnSpan)" Duration="0">
                                        <DiscreteObjectKeyFrame KeyTime="0">
                                            <DiscreteObjectKeyFrame.Value>
                                                <sys:Int32>2</sys:Int32>
                                            </DiscreteObjectKeyFrame.Value>
                                        </DiscreteObjectKeyFrame>
                                    </ObjectAnimationUsingKeyFrames>
                                    <ObjectAnimationUsingKeyFrames Storyboard.TargetName="DockResizer" Storyboard.TargetProperty="Placement" Duration="0">
                                        <DiscreteObjectKeyFrame KeyTime="0">
                                            <DiscreteObjectKeyFrame.Value>
                                                <Dock>Top</Dock>
                                            </DiscreteObjectKeyFrame.Value>
                                        </DiscreteObjectKeyFrame>
                                    </ObjectAnimationUsingKeyFrames>
                                    <DoubleAnimation BeginTime="00:00:00" From="4" To="4" Storyboard.TargetName="DockResizer" Storyboard.TargetProperty="MinHeight"/>
                                </Storyboard>
                            </VisualState>
                            <VisualState x:Name="HideResizer"/>
                        </VisualStateGroup>
                        <VisualStateGroup x:Name="DropDownDisplayStates">
                            <VisualState x:Name="DropDownButtonCollapsed"/>
                            <VisualState x:Name="DropDownButtonVisible">
                                <Storyboard>
                                    <ObjectAnimationUsingKeyFrames Storyboard.TargetName="DropDownButtonElement" Storyboard.TargetProperty="Visibility">
                                        <DiscreteObjectKeyFrame KeyTime="0">
                                            <DiscreteObjectKeyFrame.Value>
                                                <Visibility>Visible</Visibility>
                                            </DiscreteObjectKeyFrame.Value>
                                        </DiscreteObjectKeyFrame>
                                    </ObjectAnimationUsingKeyFrames>
                                </Storyboard>
                            </VisualState>
                        </VisualStateGroup>
                    </VisualStateManager.VisualStateGroups>
                    <Grid.RowDefinitions>
                        <RowDefinition Height="Auto"/>
                        <RowDefinition Height="Auto"/>
                        <RowDefinition Height="*"/>
                    </Grid.RowDefinitions>
                    <Grid.ColumnDefinitions>
                        <ColumnDefinition Width="Auto"/>
                        <ColumnDefinition Width="*"/>
                    </Grid.ColumnDefinitions>
                    <dock:RadGridResizer x:Name="DockResizer" Placement="Left" ShowsPreview="True" Visibility="Collapsed"/>
                    <Grid x:Name="ItemsContainer" Grid.Row="1" Grid.Column="1">
                        <Grid.ColumnDefinitions>
                            <ColumnDefinition Width="Auto"/>
                            <ColumnDefinition Width="*"/>
                            <ColumnDefinition Width="Auto"/>
                            <ColumnDefinition Width="Auto"/>
                            <ColumnDefinition Width="Auto"/>
                        </Grid.ColumnDefinitions>
                        <RepeatButton x:Name="LeftScrollButtonElement" Visibility="{Binding ComputedHorizontalScrollBarVisibility, ElementName=ScrollViewerElement}" Style="{StaticResource RepeatButtonScrollLeftStyle}"/>
                        <ScrollViewer x:Name="ScrollViewerElement" Grid.Column="1" HorizontalScrollBarVisibility="{Binding (ScrollViewer.HorizontalScrollBarVisibility), RelativeSource={RelativeSource TemplatedParent}}">
                            <ScrollViewer.Template>
                                <ControlTemplate TargetType="ScrollViewer">
                                    <ScrollContentPresenter x:Name="ScrollContentPresenter"/>
                                </ControlTemplate>
                            </ScrollViewer.Template>
                            <ItemsPresenter x:Name="ItemsPresenterElement"/>
                        </ScrollViewer>
                        <RepeatButton x:Name="RightScrollButtonElement"
                        Grid.Column="2"
                        Visibility="{Binding ComputedHorizontalScrollBarVisibility, ElementName=ScrollViewerElement}"
                        Style="{StaticResource RepeatButtonScrollRightStyle}"/>
                        <telerik:RadDropDownButton x:Name="DropDownButtonElement"
                        Grid.Column="3"
                        Visibility="Collapsed"
                        IsOpen="{Binding IsDropDownOpen, Mode=TwoWay, RelativeSource={RelativeSource TemplatedParent}}"
                        DropDownPlacement="Bottom"
                        DropDownButtonPosition="Left"
                        Style="{StaticResource DropDownButtonDocumentDropDownStyle}"
                        Focusable="False">
                            <telerik:RadDropDownButton.DropDownContent>
                                <telerikTabControl:DropDownMenu x:Name="DropDownMenuElement"
                                StaysOpen="False"
                                IconColumnWidth="0"
                                Placement="Bottom"
                                ItemTemplate="{TemplateBinding ItemDropDownContentTemplate}"
                                ItemTemplateSelector="{TemplateBinding ItemDropDownContentTemplateSelector}"
                                DisplayMemberPath="{TemplateBinding DropDownDisplayMemberPath}"/>
                            </telerik:RadDropDownButton.DropDownContent>
                        </telerik:RadDropDownButton>
                        <telerik:RadButton x:Name="CloseButton"
                        Grid.Column="4"
                        Command="telerik:RadDockingCommands.Close"
                        Style="{StaticResource CustomRadButtonDocumentCloseStyle}"
                        Visibility="{Binding IsEnabled, RelativeSource={RelativeSource Self}, Converter={StaticResource BooleanToVisibilityConverter}}"/>
                    </Grid>
                    <Grid Grid.Row="2" Grid.Column="1" Background="{TemplateBinding Background}" Margin="0 -1 0 0">
                        <Rectangle StrokeThickness="1" Stroke="{telerik:Office2013Resource ResourceKey=HighDarkBrush}"/>
                        <Rectangle Stroke="{telerik:Office2013Resource ResourceKey=AccentBrush}" StrokeThickness="1" Visibility="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=SelectedPane.IsActive, Converter={StaticResource BooleanToVisibilityConverter}}"/>
                        <Rectangle Margin="1" StrokeThickness="3" Stroke="{telerik:Office2013Resource ResourceKey=AccentBrush}" Opacity="0.3"/>
                        <Grid Margin="5">
                            <ContentPresenter x:Name="ContentElement"
                            Visibility="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=IsContentPreserved, Converter={StaticResource InvertedBooleanToVisibilityConverter}}"
                            ContentTemplate="{TemplateBinding SelectedContentTemplate}"
                            ContentTemplateSelector="{TemplateBinding ContentTemplateSelector}"/>
                            <Grid x:Name="ContentElementsPanel" Visibility="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=IsContentPreserved, Converter={StaticResource BooleanToVisibilityConverter}}"/>
                        </Grid>
                    </Grid>
                </Grid>
            </ControlTemplate>
            <Style BasedOn="{StaticResource RadPaneGroupStyle}" TargetType="telerik:RadPaneGroup">
                <Setter Property="DocumentHostTemplate" Value="{StaticResource RadPaneGroupDocumentHostControlTemplate}"/>
            </Style>