The error is "Wrong Document". The dialog creates the table from different document instead of the document of the RadEditor's content area.

Here is the workaround:

<script type="text/javascript">
	Telerik.Web.UI.InsertTableLight.prototype.getModifiedTable = function(tableToModify){
		var rowCount = parseInt(this._rowsCount.value, 10);
		var colCount = parseInt(this._colsCount.value, 10);
		if (!tableToModify && !(isNaN(rowCount) || isNaN(colCount))) {
			tableToModify = this._editor.get_document().createElement("table");
			this._originalRowsCount = 0;
			this._originalColsCount = 0;
		}

		if (!tableToModify) return;

		var originalRowsCount = this._originalRowsCount;
		var originalColsCount = this._originalColsCount;

		if (rowCount > 0 && rowCount != originalRowsCount) {
			rowCount > originalRowsCount ? this.addRowsCols(tableToModify, originalRowsCount, rowCount, true) : this.removeRowsCols(tableToModify, originalRowsCount, rowCount, true);
		}
		if (colCount > 0 && colCount != originalColsCount) {
			colCount > originalColsCount ? this.addRowsCols(tableToModify, originalColsCount, colCount, false) : this.removeRowsCols(tableToModify, originalColsCount, colCount, false);
		}
		this._setAttribValue(tableToModify, "align", this._alignmentSelectorTable.getAlign());
		var oSpacing = parseInt(this._cellSpacing.value, 10);
		if (!isNaN(parseInt(oSpacing, 10))) this._setAttribValue(tableToModify, "cellSpacing", oSpacing >= 0 ? oSpacing : "", (oSpacing >= 0));

		var oPadding = parseInt(this._cellPadding.value, 10);
		if (!isNaN(parseInt(oPadding, 10))) this._setAttribValue(tableToModify, "cellPadding", oPadding >= 0 ? oPadding : "", (oPadding >= 0));

		var oBorderWidth = parseInt(this._borderWidth.value, 10);
		if (!isNaN(parseInt(oBorderWidth, 10))) this._setAttribValue(tableToModify, "border", oBorderWidth >= 0 ? oBorderWidth : "", (oBorderWidth >= 0));

		return tableToModify;
	}
</script>