Here's an example of a combo box, the others work similarly.
<DataGrid.Columns>
<datagridtemplatecolumn.celleditingtemplate>
<datatemplate>
<combobox height="22" name="cmbSelectedColumn">
SelectedValue="{Binding ColumnId}"
SelectedValuePath="Id"
DisplayMemberPath="Name"
ItemsSource="{DynamicResource AvailColumns}"
SelectionChanged="cmbSelectedColumn_SelectionChanged" />
</combobox></datatemplate>
</datagridtemplatecolumn.celleditingtemplate>