Creating a custom product sort option in woocommerce categories by meta if exists

I am able to create a custom product sorting option using the official woo documentation. However, when chosen, only products that have the custom meta appear. I'd like to add order by modified time if the custom meta does not exist for a product. The end result would be products with the custom meta will appear on top sorted, all products without the custom meta will appear after them sorted by modified time.

This is the function:

add_filter( 'woocommerce_get_catalog_ordering_args', 'custom_woocommerce_get_catalog_ordering_args' );
function custom_woocommerce_get_catalog_ordering_args( $args ) {
  $orderby_value = isset( $_GET['orderby'] ) ? wc_clean( $_GET['orderby'] ) : apply_filters( 'woocommerce_default_catalog_orderby', get_option( 'woocommerce_default_catalog_orderby' ) );
    if ( 'custom_meta' == $orderby_value ) {
        $args['orderby'] = 'custom_meta';
        $args['order'] = 'DESC';
        $args['meta_key'] = 'meta_key_num';
    }
    return $args;
}
add_filter( 'woocommerce_default_catalog_orderby_options', 'custom_woocommerce_catalog_orderby' );
add_filter( 'woocommerce_catalog_orderby', 'custom_woocommerce_catalog_orderby' );
function custom_woocommerce_catalog_orderby( $sortby ) {
    $sortby['custom_meta'] = 'Custom';
    return $sortby;
}