When a Case status is updated to Closed, check if there are any open child WorkOrder or Task records; if yes, throw a validation error using addError().

trigger CaseTrigger on Case (before update) {
    Set<Id> closing = new Set<Id>();
    for (Case c : Trigger.new) if (c.Status == 'Closed' && Trigger.oldMap.get(c.Id).Status != 'Closed') closing.add(c.Id);
    if (closing.isEmpty()) return;
    Set<Id> blocked = new Set<Id>();
    for (WorkOrder w : [SELECT CaseId FROM WorkOrder WHERE CaseId IN :closing AND IsClosed = false]) blocked.add(w.CaseId);
    for (Task t : [SELECT WhatId FROM Task WHERE WhatId IN :closing AND IsClosed = false]) blocked.add(t.WhatId);
    for (Case c : Trigger.new) {
        if (blocked.contains(c.Id)) c.addError('Close all open Work Orders and Tasks before closing the Case.');
    }
}