In RadGridView with a TextBoxColumn.
  Set WrapText, Multiline and AcceptsReturn to true
  If  adding a new line, the return gives another new line instead of staying within the cell.

Workaround:
  Create a new behavior class:
//VB
Class MyNewRowBehavior
    Inherits GridNewRowBehavior
    Protected Overrides Function ProcessEnterKey(keys As KeyEventArgs) As Boolean
        If Me.GridControl.IsInEditMode AndAlso Me.GridControl.CurrentColumn.Name = "TextBoxColumn" Then
            Dim editor As RadTextBoxEditor = TryCast(Me.GridControl.ActiveEditor, RadTextBoxEditor)
            Dim element As RadTextBoxEditorElement = DirectCast(editor.EditorElement, RadTextBoxEditorElement)

            element.Text.Insert(element.Text.Length, Environment.NewLine)
            Return True
        Else
            Return MyBase.ProcessEnterKey(keys)
        End If
    End Function
End Class

//C#
class MyNewRowBehavior : GridNewRowBehavior
{
    protected override bool ProcessEnterKey(KeyEventArgs keys)
    {
        if (this.GridControl.IsInEditMode && this.GridControl.CurrentColumn.Name == "TextBoxColumn")
        {
            RadTextBoxEditor editor = this.GridControl.ActiveEditor as RadTextBoxEditor;
            RadTextBoxEditorElement element = (RadTextBoxEditorElement)editor.EditorElement;

            element.Text.Insert(element.Text.Length, Environment.NewLine);
            return true;
        }
        else
        {
            return base.ProcessEnterKey(keys);
        }
    }
}

then unregister the old behavior and register the new one:
//VB
DirectCast(radGridView1.GridBehavior, BaseGridBehavior).UnregisterBehavior(GetType(GridViewNewRowInfo))
DirectCast(radGridView1.GridBehavior, BaseGridBehavior).RegisterBehavior(GetType(GridViewNewRowInfo), New MyNewRowBehavior())

//C#
((BaseGridBehavior)radGridView1.GridBehavior).UnregisterBehavior(typeof(GridViewNewRowInfo));
((BaseGridBehavior)radGridView1.GridBehavior).RegisterBehavior(typeof(GridViewNewRowInfo), new MyNewRowBehavior());