Solved by using httpErrors in web.config. The site now redirects to the custom error page and does not prompt for windows credentials 3 times.
<system.webServer>
<httpErrors errorMode="DetailedLocalOnly" defaultResponseMode="File">
<remove statusCode="401" />
<error statusCode="401" path="/AccessDenied.aspx" responseMode="ExecuteURL" />
</httpErrors>
</system.webServer>