To reproduce, add menu item in the filtering context menu and upon click, set it as checked. Note that the filtering cell text should also take the custom menu item text or there should be an approach to set it.
Workaround: uncomment the commented code below.

protected override void OnLoad(EventArgs e)
        {
            base.OnLoad(e);

            AddGrid();

            DataTable dt = new DataTable();
            dt.Columns.Add("Id", typeof(int));
            dt.Columns.Add("Name", typeof(string));

            for (int i = 0; i < 5; i++)
            {
                dt.Rows.Add(i, "Item" + i);
            }
            dt.Rows.Add(5, null);
            dt.Rows.Add(6, "");

            radGridView1.ShowHeaderCellButtons = true;
            this.radGridView1.DataSource = dt;
            this.radGridView1.AutoSizeColumnsMode = Telerik.WinControls.UI.GridViewAutoSizeColumnsMode.Fill;
            this.radGridView1.EnableFiltering = true;
            this.radGridView1.EnableCustomFiltering = true;
            //radGridView1.ContextMenuOpening += radGridView1_ContextMenuOpening;
        }

        private void radGridView1_ContextMenuOpening(object sender, Telerik.WinControls.UI.ContextMenuOpeningEventArgs e)
        {
            GridFilterCellElement filterCell = e.ContextMenuProvider as GridFilterCellElement;
            GridViewDataColumn dataCol = (GridViewDataColumn)filterCell.ColumnInfo;
            if (filterCell != null)
            {
                if (filterCell.ColumnInfo.Name == "Name")
                {
                    RadMenuItem isNullMenuItem = new RadMenuItem();
                    isNullMenuItem.Click += customMenuItem_Click;
                    isNullMenuItem.Text = "My IsNull";
                    isNullMenuItem.Tag = filterCell.ColumnInfo.Name;
                    e.ContextMenu.Items.Add(isNullMenuItem);

                    RadMenuItem isNotNullMenuItem = new RadMenuItem();
                    isNotNullMenuItem.Click += customMenuItem_Click;
                    isNotNullMenuItem.Text = "My IsNotNull";
                    isNotNullMenuItem.Tag = filterCell.ColumnInfo.Name;
                    e.ContextMenu.Items.Add(isNotNullMenuItem);

                    bool isCustomFilter = (dataCol.FilterDescriptor != null &&
                        dataCol.FilterDescriptor is CompositeFilterDescriptor &&
                        dataCol.FilterDescriptor.Expression.Contains("NULL"));

                    foreach (RadMenuItem item in e.ContextMenu.Items)
                    {
                        if (item.Text == "Is null" || item.Text == "Is not null")
                        {
                            item.Visibility = Telerik.WinControls.ElementVisibility.Collapsed;
                        }
                        //if (isCustomFilter)
                        //{
                        //    item.IsChecked = false;
                        //}
                    }
                    //if (isCustomFilter)
                    //{
                    //    if (((CompositeFilterDescriptor)dataCol.FilterDescriptor).LogicalOperator == FilterLogicalOperator.And)
                    //    {
                    //        isNotNullMenuItem.IsChecked = true;
                    //    }
                    //    else
                    //    {
                    //        isNullMenuItem.IsChecked = true;
                    //    }
                    //}
                }
            }
        }

        private void customMenuItem_Click(object sender, EventArgs e)
        {
            RadMenuItem clickedItem = (RadMenuItem)sender;
            string columnName = clickedItem.Tag.ToString();
            radGridView1.Columns[columnName].FilterDescriptor = null;

            FilterOperator filterOperator = clickedItem.Text.Contains("Not") ? FilterOperator.IsNotEqualTo : FilterOperator.IsEqualTo;
            FilterLogicalOperator logicalOperator = clickedItem.Text.Contains("Not") ? FilterLogicalOperator.And : FilterLogicalOperator.Or;

            CompositeFilterDescriptor compositeFilter = new CompositeFilterDescriptor();
            compositeFilter.FilterDescriptors.Add(new FilterDescriptor(columnName, filterOperator, ""));
            compositeFilter.FilterDescriptors.Add(new FilterDescriptor(columnName, filterOperator, null));
            compositeFilter.LogicalOperator = logicalOperator;
            compositeFilter.IsFilterEditor = true;
            this.radGridView1.FilterDescriptors.Add(compositeFilter);

            clickedItem.IsChecked = true;
        }