Came up with a solution. This edit makes it so that if the coupon is a dollar amount AND is restricted to one product AND the customer is in a discount group, the coupon will deduct the value of the coupon MINUS the discount group percentage.
This allows me to offer a full dollar amount coupon that is restricted to a single product and have it adjust the price if the customer is in a discount group. For example, a coupon for $10 off a $10 product will adjust to $9.70 if the customer gets a 3% discount, otherwise it will be $10.
If you want to make sure it is the full amount, you can just restrict the coupon to a second product that is disabled. For example, if you want a coupon for $5 off a $10 product and you want group discount members to get the entire $5, you would restrict the coupon to the eligible product, then add a second restriction for a disabled product.
Percentage coupons will be deducted from the subtotal AFTER the group discount (as long as your coupon order total module is after the group discount module).
} elseif ($coupon->fields['coupon_type'] == 'F') {
//IF customer is in a discount group
//AND coupon code is dollar amount (not percentage)
//AND coupon is only valid on one item
//THEN coupon amount is the total coupon minus group discount
//ELSE coupon amount is 100% of coupon value
$group_query = $db->Execute("select customers_group_pricing from " . TABLE_CUSTOMERS . " where customers_id = '" . (int)$_SESSION['customer_id'] . "'");
$cr_query = $db->Execute("select count(product_id) as total_restrictions from " . TABLE_COUPON_RESTRICT . " where coupon_id = '" . (int)$coupon->fields['coupon_id'] . "' and coupon_restrict = 'N' and category_id = '0'");
if ($group_query->fields['customers_group_pricing'] != '0' && (int)$cr_query->fields['total_restrictions'] == 1) {
$group_percentage_query = $db->Execute("select group_percentage from " . TABLE_GROUP_PRICING . "
where group_id = '" . (int)$group_query->fields['customers_group_pricing'] . "'");
$coupon->fields['coupon_amount'] -= $coupon->fields['coupon_amount'] * $group_percentage_query->fields['group_percentage']/100;
}
$od_amount['total'] = round($coupon->fields['coupon_amount'] * ($orderTotal>0), 3);
$od_amount['type'] = 'F';
$ratio = $od_amount['total']/$orderTotal;
}