Your javascript function does not return true value. So, onClick event will never active. after returning true in OnClientClick, onClick event occur.
Like
function filterSearch()
{
var combo = $find('<%=RadComboBox1.ClientID %>');
var comboValue = document.getElementById(combo.UniqueID + "_value").value;
if (comboValue == null)
{
alert("You must select a category before processing search!");
return false;
}
return true;
}
<asp:button id="btnSearch" runat="server" style="float: left; color: black; font-weight: bold" text="Search" onclick="btnSearch_Click" onclientclick="return filterSearch()==true?true:false;" xmlns:asp="#unknown" />