You have to register keypress event on page load. You can place this code in master page ans add all 'ModalPopupExtenderBehaviorID' in 'if' condition.
function my_onload() {
window.addEventListener("keypress", function (event) { myExtension_with_keypress.init(event); }, false);
var myExtension_with_keypress = {
init: function (event) {
if (event.keyCode == 27) {
$find('ModalPopupExtenderBehaviorID').hide();
return false;
}
}
}
}
window.onload = my_onload;