generalized approach to pulling named elements from deep within a nested list

Viewed 103

I asked a question yesterday asking for how to access all elements of a nested list by name. https://stackoverflow.com/a/64579426/9806500

Is there a generalized function that will let me get named elements that are deeper in the list? In particular, I'm trying to get the users and timestamps of replies to a Slack message, and the message schema names reuses user and ts under a replies level of the message in pairs to designate who and when replied. I cannot get to them with the purrr solution link above. Not sure if this is because user and ts are also element names for the main message, or because I'm not able to access levels that far down by using ~ purrr::map() as a mapper function inside map().

l <- list(folder_1 = list(
      `msg_1-1` = list(type = "message",
                   subtype = "channel_join",
                   ts = "1585771048.000200",
                   user = "UFUNNF8MA",
                   text = "<@UFUNNF8MA> has joined the channel"),
      `msg_1-2` = list(type = "message",
                       subtype = "channel_purpose",
                       ts = "1585771049.000300",
                       user = "UNFUNQ8MA",
                       text = "<@UNFUNQ8MA> set the channel purpose: Talk about xyz")),
      folder_2 = list(
        `msg_2-1` = list(type = "message",
                      subtype = "channel_join",
                      ts = "1585771120.000200",
                      user = "UQKUNF8MA",
                      text = "<@UQKUNF8MA> has joined the channel",
                      replies = list(list(user = "UABCDEFG8", ts = "1585771220.002200"),
                          list(user = "UGFEDCBA8", ts = "1585771220.022000")))))

Thanks!

4 Answers

You could do:

unlist(l)[grep("\\.user", names(unlist(l)))]
#>         folder_1.msg_1-1.user         folder_1.msg_1-2.user 
#>                   "UFUNNF8MA"                   "UNFUNQ8MA" 
#>         folder_2.msg_2-1.user folder_2.msg_2-1.replies.user 
#>                   "UQKUNF8MA"                   "UABCDEFG8" 
#> folder_2.msg_2-1.replies.user 
#>                   "UGFEDCBA8" 

For a more general solution, you could have:

 get_elements <- function(list, field) {
   as.character(unlist(list)[grep(paste0(field, "$"), names(unlist(list)))])
 }

Which allows:

get_elements(l, "user")
#> [1] "UFUNNF8MA" "UNFUNQ8MA" "UQKUNF8MA" "UABCDEFG8" "UGFEDCBA8"

get_elements(l, "ts")
#> [1] "1585771048.000200" "1585771049.000300" "1585771120.000200"

You can use this recursive function:

# assign the function
get_ele <- function(x, what){
  is_list <- vapply(x, is.list, FALSE)
  c(sapply(x[is_list], get_ele, what), x[[what]])
}

# use the function
unname(unlist(get_ele(l, "user")))
#R> [1] "UFUNNF8MA" "UNFUNQ8MA" "UABCDEFG8" "UGFEDCBA8" "UQKUNF8MA"

unname(unlist(get_ele(l, "ts")))
#R> [1] "1585771048.000200" "1585771049.000300" "1585771120.000200"

And if you need to do this a lot of times, then this C++ version might be faster:

#include <Rcpp.h>
#include <vector>
using namespace Rcpp;

void get_ele_inner(Rcpp::List x, Rcpp::CharacterVector what, 
                   std::vector<SEXP> &out_list){
  SEXP names = x.attr("names");
  if(!Rf_isNull(names)){
    CharacterVector names_str(names);
    for(R_len_t i = 0; i < x.size(); ++i){
      if(names_str[i] == what[0])
        out_list.emplace_back(x[i]);
      else if(Rf_isNewList(x[i]))
        get_ele_inner(x[i], what, out_list);
      
    }
  } else 
    for(auto xi : x)
      if(Rf_isNewList(xi))
        get_ele_inner(xi, what, out_list);
}

// [[Rcpp::export(rng = false)]]
SEXP get_ele(SEXP x, Rcpp::CharacterVector what) {
  std::vector<SEXP> out_list;
  get_ele_inner(x, what, out_list);
  
  R_len_t const n_ele = out_list.size();
  Rcpp::List out(n_ele);
  for(R_len_t i = 0; i < n_ele; ++i)
    out[i] = out_list[i];
  
  return out;
}

You can use Rcpp::sourceCpp to compile the function. It seems 10 times faster on my laptop with the list you provide (though this list is quite small).

A "tidy" solution:

# Fixed the reprex list
l <- list(
  folder_1 = list(
    `msg_1-1` = list(
      type = "message",
      subtype = "channel_join",
      ts = "1585771048.000200",
      user = "UFUNNF8MA",
      text = "<@UFUNNF8MA> has joined the channel"
    ),
    `msg_1-2` = list(
      type = "message",
      subtype = "channel_purpose",
      ts = "1585771049.000300",
      user = "UNFUNQ8MA",
      text = "<@UNFUNQ8MA> set the channel purpose: Talk about xyz"
    )
  ),
  folder_2 = list(
    `msg_2-1` = list(
      type = "message",
      subtype = "channel_join",
      ts = "1585771120.000200",
      user = "UQKUNF8MA",
      text = "<@UQKUNF8MA> has joined the channel",
      replies = list(
        list(user = "UABCDEFG8", ts = "1585771220.002200"),
        list(user = "UGFEDCBA8", ts = "1585771220.022000")
      )
    )
  )
)
# use the pipe
`%>%` <- magrittr::`%>%`
# map at 2nd level depth
messages <- purrr::map_depth(l, 2, ~{
  if (!is.null(.x$replies)) {
#Map over replies if present
    .replies <- purrr::map_dfr(.x$replies, ~{
      tibble::tibble(ts = .x$ts, user = .x$user, text = rlang::`%||%`(.x$text, NA))
    })
  } else {
    .replies <- NULL
  }
#output as tibble
  tibble::tibble(
    ts = .x$ts, user = .x$user, text = .x$text, replies =  list(
      .replies
    )
  )
  
}) %>%
#remove one layer of nesting
  unlist(recursive = FALSE) %>%
#bind together and keep folder name
  dplyr::bind_rows(.id = "folder") %>% 
# separate message name into it's own column
  tidyr::separate(folder, into = c("folder", "message"), sep = "\\.")

Using rrapply() in the rrapply-package (an extended version of base rapply()):

library(rrapply)

## unlist 'user' nodes as vector
rrapply(l, condition = function(x, .xname) .xname == "user", how = "unlist")
#>         folder_1.msg_1-1.user         folder_1.msg_1-2.user 
#>                   "UFUNNF8MA"                   "UNFUNQ8MA" 
#>         folder_2.msg_2-1.user folder_2.msg_2-1.replies.user 
#>                   "UQKUNF8MA"                   "UABCDEFG8" 
#> folder_2.msg_2-1.replies.user 
#>                   "UGFEDCBA8"

## unlist 'ts' nodes as vector
rrapply(l, condition = function(x, .xname) .xname == "ts", how = "unlist")
#>         folder_1.msg_1-1.ts         folder_1.msg_1-2.ts 
#>         "1585771048.000200"         "1585771049.000300" 
#>         folder_2.msg_2-1.ts folder_2.msg_2-1.replies.ts 
#>         "1585771120.000200"         "1585771220.002200" 
#> folder_2.msg_2-1.replies.ts 
#>         "1585771220.022000"

Or returning melted data.frames instead of unlisted vectors:

## 'user' nodes as melted data.frame
rrapply(l, condition = function(x, .xname) .xname == "user", how = "melt")
#>         L1      L2      L3   L4   L5     value
#> 1 folder_1 msg_1-1    user <NA> <NA> UFUNNF8MA
#> 2 folder_1 msg_1-2    user <NA> <NA> UNFUNQ8MA
#> 3 folder_2 msg_2-1    user <NA> <NA> UQKUNF8MA
#> 4 folder_2 msg_2-1 replies  ..1 user UABCDEFG8
#> 5 folder_2 msg_2-1 replies  ..2 user UGFEDCBA8

## 'ts' nodes as melted data.frame
rrapply(l, condition = function(x, .xname) .xname == "ts", how = "melt")
#>         L1      L2      L3   L4   L5             value
#> 1 folder_1 msg_1-1      ts <NA> <NA> 1585771048.000200
#> 2 folder_1 msg_1-2      ts <NA> <NA> 1585771049.000300
#> 3 folder_2 msg_2-1      ts <NA> <NA> 1585771120.000200
#> 4 folder_2 msg_2-1 replies  ..1   ts 1585771220.002200
#> 5 folder_2 msg_2-1 replies  ..2   ts 1585771220.022000
Related