Here's the answer :
public List<Orders> custOrder() {
try ( Session session = HibernateUtil.getSessionFactory().openSession()) {
CriteriaBuilder cb = session.getCriteriaBuilder();
CriteriaQuery<Orders> cq = cb.createQuery(Orders.class);
Root<Orders> root = cq.from(Orders.class);
Subquery<Long> subquery = cq.subquery(Long.class);
Root<Orders> subRoot = subquery.from(Orders.class);
subquery.select(cb.max(subRoot.get("id")));
subquery.groupBy(subRoot.get("transid"));
cq.where(cb.in(root.get("id")).value(subquery));
return session.createQuery(cq).getResultList();
}
}