Set a count variable and check. If value is greater than 10 then show the message.
Try this:
$("[id$='SubmitBtn']").click(function () {
var ISBN = [];
var count = 0;
$("[id$='LocationDataList'] input[type=checkbox]:checked").each(function () {
ISBN.push($(this).val());
count = count + 1;
});
if(count > 10){
alert('You selected more than 10 locations');
return false;
}
else{
$("[id$='HiddenField1']").val(ISBN);
return true;
}
});
--Amit