To reproduce:

private void Form1_Load(object sender, EventArgs e) 
{ 
    this.productsTableAdapter.Fill(this.nwindDataSet.Products);
    this.radGridView1.EnableFiltering = true;
    this.radGridView1.ShowHeaderCellButtons = true;
    this.radGridView1.ShowFilteringRow = false;

    this.radGridView1.Columns["UnitPrice"].FormatString = "{0:$#,###0.00;-$#,###0.00;$0.00}";
}

Workaround:  in order to format the cells, use the CellFormatting event. As to the filter popup, use the following approach:
 this.radGridView1.FilterPopupInitialized += radGridView1_FilterPopupInitialized;

private void radGridView1_FilterPopupInitialized(object sender, Telerik.WinControls.UI.FilterPopupInitializedEventArgs e)
{
    RadListFilterPopup popup = e.FilterPopup as RadListFilterPopup;
    if (popup != null)
    {
        popup.MenuTreeElement.TreeView.NodeFormatting -= TreeView_NodeFormatting;
        popup.MenuTreeElement.TreeView.NodeFormatting += TreeView_NodeFormatting;
    }
}

private void TreeView_NodeFormatting(object sender, TreeNodeFormattingEventArgs e)
{
    decimal price;
    if (decimal.TryParse(e.Node.Text, out price))
    {
        e.NodeElement.ContentElement.Text = string.Format("{0:$#,###0.00;-$#,###0.00;$0.00}", price);
    }
}