A workaround is to set the table element's height as well:

<telerik:RadEditor runat="server" ID="RadEditor1" Height="400">
</telerik:RadEditor>

<telerik:RadButton runat="server" ID="RadBtn1" AutoPostBack="false" 
    Text="Change Size" OnClientClicked="changeEditorSize"></telerik:RadButton>  

<script type="text/javascript">
    function changeEditorSize(sender, args) {
        var editor = $find("<%= RadEditor1.ClientID %>");
        var width = editor.get_element().style.width;
        editor.setSize(width, "200px");
        //this line is the workaround
        editor.get_mainTable().style.height = "200px";
    }
</script>