Item's GroupSettings are taken into account during the child items rendering, so the following statement: "menu.get_items().getItem(0).get_groupSettings().set_repeatColumns(2); " should be executed prior to adding child items.

        <script type="text/javascript">
            function pageLoad() {
                AddNewItems();
            }

            function AddNewItems() {
                var menu = $find("<%= RadMenu1.ClientID %>");
                var rootItem = new Telerik.Web.UI.RadMenuItem();
                rootItem.set_text("Root1");
                var firstChild = new Telerik.Web.UI.RadMenuItem();
                firstChild.set_text("Child1");
                var secondChild = new Telerik.Web.UI.RadMenuItem();
                secondChild.set_text("Child2");
                var thirdChild = new Telerik.Web.UI.RadMenuItem();
                thirdChild.set_text("Child3");
                var fourthChild = new Telerik.Web.UI.RadMenuItem();
                fourthChild.set_text("Child4");

                menu.trackChanges();

                menu.get_items().add(rootItem);
                menu.get_items().getItem(0).get_groupSettings().set_repeatColumns(2);

                rootItem.get_items().add(firstChild);
                rootItem.get_items().add(secondChild);
                rootItem.get_items().add(thirdChild);
                rootItem.get_items().add(fourthChild);

                menu.commitChanges();
            }
        </script>