Currently RadButton configured as a toggle radio/checkbox button doesn't accept custom icons to be used because it already has predefined icons. For the time being the ButtonType property can be set to "StandardButton" and its background image can be hidden. For example:
CSS:
<style>
    .RadButton_Default.rbSkinnedButton,
    .RadButton_Default .rbDecorated,
    .RadButton_Default.rbVerticalButton,
    .RadButton_Default.rbVerticalButton .rbDecorated,
    .RadButton_Default .rbSplitRight,
    .RadButton_Default .rbSplitLeft {
        background-image: none !important;
    }
</style>
ASPX:
<telerik:RadButton ID="RadButton3" runat="server" GroupName="OverSampling"
    Text="" ButtonType="StandardButton" ToggleType="Radio" Checked="false" AutoPostBack="false">
    <ToggleStates>
        <telerik:RadButtonToggleState Text="Yes" PrimaryIconUrl="radio6-on.png" Selected="true" />
        <telerik:RadButtonToggleState Text="Yes" PrimaryIconUrl="radio6-off.png" />
    </ToggleStates>
</telerik:RadButton>
 
<telerik:RadButton ID="RadButton4" runat="server" GroupName="OverSampling"
    Text="No" ButtonType="StandardButton" ToggleType="Radio" AutoPostBack="false" Checked="true">
    <ToggleStates>
        <telerik:RadButtonToggleState PrimaryIconUrl="radio6-on.png" Selected="true" />
        <telerik:RadButtonToggleState PrimaryIconUrl="radio6-off.png" />
    </ToggleStates>
</telerik:RadButton>