public class MyPdfViewer : RadPdfViewer
{
    public PdfDocumentSource PdfDocumentSource
    {
        get { return (PdfDocumentSource)GetValue(PdfDocumentSourceProperty); }
        set { SetValue(PdfDocumentSourceProperty, value); }
    }

    public static readonly DependencyProperty PdfDocumentSourceProperty =
        DependencyProperty.Register("PdfDocumentSource", typeof(PdfDocumentSource), typeof(MyPdfViewer), new PropertyMetadata(new PropertyChangedCallback(OnDocumentSourceChanged)));

    private static void OnDocumentSourceChanged(DependencyObject d, DependencyPropertyChangedEventArgs e)
    {
        MyPdfViewer viewer = (MyPdfViewer)d;

        viewer.SetDocumentSource((PdfDocumentSource)e.NewValue);
    }
}

<local:MyPdfViewer x:Name="pdfViewer" PdfDocumentSource="{Binding MyViewModelPropertyOfTypePdfDocumentSource}" />