Reduce grouped data based on maximum of multiple columns

Viewed 121

I have datasets like this example, but with 1000 Inputs and 1000 Words for each Input, and 30 values for each Input x Time x Word combination (in cols Copy1..Copy30)

df = read.table(header=T, sep=",", text="
Input,Time,Word,Copy1,Copy2,Copy3,Copy30
ark,1,ark,0.00,0.00,0.00,0.00
ark,1,ad,0.00,0.00,0.00,0.00
ark,1,bark,0.00,0.00,0.00,0.00
ark,50,ark,0.00,0.10,0.05,0.00
ark,50,ad,0.00,0.05,0.03,0.00
ark,50,bark,0.07,0.06,0.00,0.00
ark,100,ark,0.00,0.17,0.55,0.00
ark,100,ad,0.00,0.03,0.11,0.00
ark,100,bark,0.05,0.20,0.00,0.00
bark,1,ark,0.00,0.00,0.00,0.00
bark,1,ad,0.00,0.00,0.00,0.00
bark,1,bark,0.00,0.00,0.00,0.00
bark,50,ark,0.00,0.03,0.09,0.00
bark,50,ad,0.00,0.05,0.03,0.00
bark,50,bark,0.2,0.75,0.00,0.00
bark,100,ark,0.00,0.08,0.32,0.00
bark,100,ad,0.00,0.03,0.11,0.00
bark,100,bark,0.21,0.60,0.00,0.00
") %>% arrange(Input,Time,Word)

df
# Input Time Word Copy1 Copy2 Copy3 Copy30
# 1    ark    1   ad  0.00  0.00  0.00      0
# 2    ark    1  ark  0.00  0.00  0.00      0
# 3    ark    1 bark  0.00  0.00  0.00      0
# 4    ark   50   ad  0.00  0.05  0.03      0
# 5    ark   50  ark  0.00  0.10  0.05      0
# 6    ark   50 bark  0.07  0.06  0.00      0
# 7    ark  100   ad  0.00  0.03  0.11      0
# 8    ark  100  ark  0.00  0.17  0.55      0
# 9    ark  100 bark  0.05  0.20  0.00      0
# 10  bark    1   ad  0.00  0.00  0.00      0
# 11  bark    1  ark  0.00  0.00  0.00      0
# 12  bark    1 bark  0.00  0.00  0.00      0
# 13  bark   50   ad  0.00  0.05  0.03      0
# 14  bark   50  ark  0.00  0.03  0.09      0
# 15  bark   50 bark  0.20  0.75  0.00      0
# 16  bark  100   ad  0.00  0.03  0.11      0
# 17  bark  100  ark  0.00  0.08  0.32      0
# 18  bark  100 bark  0.21  0.60  0.00      0

I want to group by Input and Word, and for each combination, determine which Copy column has the maximum value for each word, and then keep only that column for that Word for that Input. A response to a previous question got me part of the way there. This code identifies which Copy of each Word is the maximum.

max_copy <- df %>% 
  pivot_longer(starts_with("Copy"), names_to="copy_name", values_to="copy_value") %>% 
  group_by(Input, Word) %>% 
  filter(rank(copy_value, ties.method="first") == n()) %>%
  group_by(Input, Time)

max_copy
# A tibble: 6 x 5
# Groups:   Input, Time [3]
# Input  Time Word  copy_name copy_value
# <fct> <int> <fct> <chr>          <dbl>
# 1 ark     100 ad    Copy3           0.11
# 2 ark     100 ark   Copy3           0.55
# 3 ark     100 bark  Copy2           0.2 
# 4 bark     50 bark  Copy2           0.75
# 5 bark    100 ad    Copy3           0.11
# 6 bark    100 ark   Copy3           0.32

Now what I want to do is use this to reduce the data to the identified copies for each word for each input, so that the result would be:

# A tibble: 18 x 5
# Groups:   Input, Time [6]
#   Input  Time Word  copy_name copy_value
#   <fct> <int> <fct> <chr>          <dbl>
#  1 ark       1 ad    Copy3          0 
#  2 ark       1 ark   Copy3          0   
#  3 ark       1 bark  Copy2          0   
#  4 ark      50 ad    Copy3          0.03 
#  5 ark      50 ark   Copy3          0.05 
#  6 ark      50 bark  Copy2          0.06
#  7 ark     100 ad    Copy3          0.11 
#  8 ark     100 ark   Copy3          0.55
#  9 ark     100 bark  Copy2          0.2 
# 10 bark      1 ad    Copy3          0 
# 11 bark      1 ark   Copy3          0   
# 12 bark      1 bark  Copy2          0   
# 13 bark     50 ad    Copy3          0.03
# 14 bark     50 ark   Copy3          0.09
# 15 bark     50 bark  Copy2          0.75
# 16 bark    100 ad    Copy3          0.11
# 17 bark    100 ark   Copy3          0.32
# 18 bark    100 bark  Copy2          0.6 

Is there a method where I can use the max_copy data to reduce df like this?

EDIT: There are problems with some solutions below. @akrun's solution breaks if there are negative values (easy to deal with) or if there are positive values in later Copies than the Copy with the maximum value (I can't see how to fix this). @AnoushiravanR's solution appears to be robust against both conditions, and so do the solutions from @AnilGoyal. Here's an updated dataset with those kinds of conditions included.

df2 = read.table(header=T, sep=",", text="
Input,Time,Word,Copy1,Copy2,Copy3,Copy30
ark,1,ark,0.00,0.00,0.00,-0.01
ark,1,ad,0.00,0.00,0.00,-0.01
ark,1,bark,0.00,0.00,0.00,-0.01
ark,1,bar,0.00,0.00,0.00,-0.01
ark,50,ark,0.00,0.10,0.05,-0.01
ark,50,ad,0.00,0.05,0.03,-0.01
ark,50,bark,0.07,0.06,0.01,-0.01
ark,50,bar,0.07,0.06,0.01,-0.01
ark,100,ark,0.00,0.17,0.55,-0.01
ark,100,ad,0.00,0.03,0.11,-0.01
ark,100,bark,0.05,0.20,0.01,-0.01
ark,100,bar,0.04,0.15,0.01,-0.01
bark,1,ark,0.00,0.00,0.00,-0.01
bark,1,ad,0.00,0.00,0.00,-0.01
bark,1,bark,0.00,0.00,0.00,-0.01
bark,1,bar,0.00,0.00,0.00,-0.01
bark,50,ark,0.00,0.03,0.09,-0.01
bark,50,ad,0.00,0.05,0.03,-0.01
bark,50,bark,0.2,0.75,0.01,0.01
bark,50,bar,0.2,0.7,0.00,-0.01
bark,100,ark,0.00,0.08,0.32,-0.01
bark,100,ad,0.00,0.03,0.11,-0.01
bark,100,bark,0.21,0.60,0.01,-0.01
bark,100,bar,0.15,0.4,0.01,-0.01
") %>% arrange(Input,Time,Word)

Desired output for df2:

# A tibble: 24 x 5
# Input  Time Word  copy_name Value
# <fct> <int> <fct> <chr>     <dbl>
# 1 ark       1 ad    Copy3      0   
# 2 ark       1 ark   Copy3      0   
# 3 ark       1 bar   Copy2      0   
# 4 ark       1 bark  Copy2      0   
# 5 ark      50 ad    Copy3      0.03
# 6 ark      50 ark   Copy3      0.05
# 7 ark      50 bar   Copy2      0.06
# 8 ark      50 bark  Copy2      0.06
# 9 ark     100 ad    Copy3      0.11
# 10 ark    100 ark   Copy3      0.55
# 11 ark    100 bar   Copy2      0.15
# 12 ark    100 bark  Copy2      0.2 
# 13 bark     1 ad    Copy3      0   
# 14 bark     1 ark   Copy3      0   
# 15 bark     1 bar   Copy2      0   
# 16 bark     1 bark  Copy2      0   
# 17 bark    50 ad    Copy3      0.03
# 18 bark    50 ark   Copy3      0.09
# 19 bark    50 bar   Copy2      0.7 
# 20 bark    50 bark  Copy2      0.75
# 21 bark   100 ad    Copy3      0.11
# 22 bark   100 ark   Copy3      0.32
# 23 bark   100 bar   Copy2      0.4 
# 24 bark   100 bark  Copy2      0.6 
3 Answers

This can be done with summarise. After reshaping to 'long' format with pivot_longer, do a group by 'Input', 'Time' Word', then summarise to create the 'copy_value' based on a condition that if all values are 0 then return 0 or else return the last non-zero value of 'copy_value'

library(dplyr)
library(tidyr)
df %>% 
  pivot_longer(cols = starts_with('Copy'), names_to = 'copy_name', 
        values_to = 'copy_value') %>% 
  group_by(Input, Time, Word) %>% 
  summarise(copy_value = if(all(copy_value == 0)) 0 
       else last(copy_value[copy_value != 0]), .groups = 'drop')

-output

# A tibble: 18 x 4
#   Input  Time Word  copy_value
# * <chr> <int> <chr>      <dbl>
# 1 ark       1 ad          0   
# 2 ark       1 ark         0   
# 3 ark       1 bark        0   
# 4 ark      50 ad          0.03
# 5 ark      50 ark         0.05
# 6 ark      50 bark        0.06
# 7 ark     100 ad          0.11
# 8 ark     100 ark         0.55
# 9 ark     100 bark        0.2 
#10 bark      1 ad          0   
#11 bark      1 ark         0   
#12 bark      1 bark        0   
#13 bark     50 ad          0.03
#14 bark     50 ark         0.09
#15 bark     50 bark        0.75
#16 bark    100 ad          0.11
#17 bark    100 ark         0.32
#18 bark    100 bark        0.6 

If we need the 'copy_name' as well, then use the same logical expression in slice to return the row that meets the condition i.e. if all values are 0, return last row (n() - doesn't matter) or get the index of last non-zero of copy_value. Now, we do a group by 'Input', 'Word' and mutate the 'copy_name' by replacing those with the corresponding 'copy_name' where the 'copy_value' is max

df %>% 
  pivot_longer(cols = starts_with('Copy'), names_to = 'copy_name', 
        values_to = 'copy_value') %>% 
  group_by(Input, Time, Word) %>%
  arrange(copy_value) %>% 
  slice(if(all(copy_value <= 0)) n() 
       else tail(which(copy_value > 0), 1))%>% 
  group_by(Input, Word) %>% 
  mutate(copy_name = copy_name[which.max(copy_value)]) %>%
  ungroup

-output

# A tibble: 18 x 5
#   Input  Time Word  copy_name copy_value
#   <chr> <int> <chr> <chr>          <dbl>
# 1 ark       1 ad    Copy3           0   
# 2 ark       1 ark   Copy3           0   
# 3 ark       1 bark  Copy2           0   
# 4 ark      50 ad    Copy3           0.03
# 5 ark      50 ark   Copy3           0.05
# 6 ark      50 bark  Copy2           0.06
# 7 ark     100 ad    Copy3           0.11
# 8 ark     100 ark   Copy3           0.55
# 9 ark     100 bark  Copy2           0.2 
#10 bark      1 ad    Copy3           0   
#11 bark      1 ark   Copy3           0   
#12 bark      1 bark  Copy2           0   
#13 bark     50 ad    Copy3           0.03
#14 bark     50 ark   Copy3           0.09
#15 bark     50 bark  Copy2           0.75
#16 bark    100 ad    Copy3           0.11
#17 bark    100 ark   Copy3           0.32
#18 bark    100 bark  Copy2           0.6 
 

Updated Solution

I have updated my solution with your new data set. I couldn't see problem with the output but if there is anything that needs modifications I would be glad to know.

library(dplyr)
library(tidyr)
library(purrr)


df2 %>%
  mutate(Copy_value = pmap_dbl(df2 %>% select(Copy1:Copy30), ~ max(c(...))),
         Copy_name = pmap(df2 %>% select(Copy1:Copy30), ~ 
                            names(c(...)[c(...) == max(c(...))]))) %>%
  unnest(Copy_name) %>% 
  group_by(Input, Word) %>%
  mutate(Copy_name = Copy_name[which.max(Copy_value)]) %>%
  distinct() %>%
  select(-c(Copy1:Copy_value)) %>%
  right_join(df2, by = c("Input", "Time", "Word")) %>%
  rowwise() %>%
  mutate(Copy_value = map_dbl(Copy_name, ~ get({.x}))) %>%
  select(-c(Copy1:Copy30))

Output This is the output of the newly-provided data set.

   Input Time Word Copy_name Copy_value
1    ark    1   ad     Copy3       0.00
2    ark    1  ark     Copy3       0.00
3    ark    1  bar     Copy2       0.00
4    ark    1 bark     Copy2       0.00
5    ark   50   ad     Copy3       0.03
6    ark   50  ark     Copy3       0.05
7    ark   50  bar     Copy2       0.06
8    ark   50 bark     Copy2       0.06
9    ark  100   ad     Copy3       0.11
10   ark  100  ark     Copy3       0.55
11   ark  100  bar     Copy2       0.15
12   ark  100 bark     Copy2       0.20
13  bark    1   ad     Copy3       0.00
14  bark    1  ark     Copy3       0.00
15  bark    1  bar     Copy2       0.00
16  bark    1 bark     Copy2       0.00
17  bark   50   ad     Copy3       0.03
18  bark   50  ark     Copy3       0.09
19  bark   50  bar     Copy2       0.70
20  bark   50 bark     Copy2       0.75
21  bark  100   ad     Copy3       0.11
22  bark  100  ark     Copy3       0.32
23  bark  100  bar     Copy2       0.40
24  bark  100 bark     Copy2       0.60

One more approach through purrr

df %>% 
  pivot_longer(cols = starts_with('Copy'), names_to = 'copy_name', 
               values_to = 'Value') %>%
  semi_join(df %>% nest(copy_name = !c(Input, Word)) %>%
              mutate(copy_name = map_chr(copy_name, 
                                    ~ names(.x)[1 + which(.x[-1] == max(.x[-1]), arr.ind = T)[2]])),
            by = c("Input", "Word", "copy_name")
            )

# A tibble: 18 x 5
   Input  Time Word  copy_name Value
   <chr> <int> <chr> <chr>     <dbl>
 1 ark       1 ad    Copy3      0   
 2 ark       1 ark   Copy3      0   
 3 ark       1 bark  Copy2      0   
 4 ark      50 ad    Copy3      0.03
 5 ark      50 ark   Copy3      0.05
 6 ark      50 bark  Copy2      0.06
 7 ark     100 ad    Copy3      0.11
 8 ark     100 ark   Copy3      0.55
 9 ark     100 bark  Copy2      0.2 
10 bark      1 ad    Copy3      0   
11 bark      1 ark   Copy3      0   
12 bark      1 bark  Copy2      0   
13 bark     50 ad    Copy3      0.03
14 bark     50 ark   Copy3      0.09
15 bark     50 bark  Copy2      0.75
16 bark    100 ad    Copy3      0.11
17 bark    100 ark   Copy3      0.32
18 bark    100 bark  Copy2      0.6

Actually, this cn be broken into two parts -

  • first is through nesting and purrr::map_chr wherein names of those copies are found where the copy value is maximum for any Time value.
df %>% nest(copy_name = !c(Input, Word)) %>%
              mutate(copy_name = map_chr(copy_name, 
                                    ~ names(.x)[1 + which(.x[-1] == max(.x[-1]), arr.ind = T)[2]]))

# A tibble: 6 x 3
  Input Word  copy_name
  <chr> <chr> <chr>    
1 ark   ad    Copy3    
2 ark   ark   Copy3    
3 ark   bark  Copy2    
4 bark  ad    Copy3    
5 bark  ark   Copy3    
6 bark  bark  Copy2
  • second part in joining pivoted data back with this data through semi_join which is actually a filtered join.

Another approach in single pipe

df %>% nest(data = !c(Input, Word)) %>%
  mutate(data = map(data, ~ .x %>% 
                      select(Time, 1+which(.x[-1] == max(.x[-1]), arr.ind = T)[2]) %>%
                      mutate(copy = names(.)[2]) %>%
                      rename_with(~'value', 2)
                    )) %>%
  unnest(data)

# A tibble: 18 x 5
   Input Word   Time value copy 
   <chr> <chr> <int> <dbl> <chr>
 1 ark   ad        1  0    Copy3
 2 ark   ad       50  0.03 Copy3
 3 ark   ad      100  0.11 Copy3
 4 ark   ark       1  0    Copy3
 5 ark   ark      50  0.05 Copy3
 6 ark   ark     100  0.55 Copy3
 7 ark   bark      1  0    Copy2
 8 ark   bark     50  0.06 Copy2
 9 ark   bark    100  0.2  Copy2
10 bark  ad        1  0    Copy3
11 bark  ad       50  0.03 Copy3
12 bark  ad      100  0.11 Copy3
13 bark  ark       1  0    Copy3
14 bark  ark      50  0.09 Copy3
15 bark  ark     100  0.32 Copy3
16 bark  bark      1  0    Copy2
17 bark  bark     50  0.75 Copy2
18 bark  bark    100  0.6  Copy2
Related