Optimum way of traversing a large dataset

user_412628 0
Viewed 70

I have a large dataset of the values as JSON. Following is a small snapshot of that JSON object. 

 {
     "response": [
     {
         "data": [
         {
             "value": 1,
             "minute": "2019-06-10 11:51",
             "action": "firstApp",
         },
         {
             "value": 10,
             "minute": "2019-06-10 11:51",
             "action": "secondApp",
         },
         {
             "value": 100,
             "minute": "2019-06-10 11:51",
             "action": "thirdApp",
         },
         {
             "value": 10,
             "minute": "2019-06-10 11:52",
             "action": "firstApp",
         },
         {
             "value": 20,
             "minute": "2019-06-10 11:52",
             "action": "secondApp",
         },
         {
             "value": 115,
             "minute": "2019-06-10 11:52",
             "action": "thirdApp",
         }, ]
     }]
 }

Now consider that there are about 800 of these items in the data array. What I would like to do is to create a JSON object which has values for each and every timestmap, much like the following

[
   {
       "timestamp" : "2019-06-10 11:51",
       "firstApp" : {
           "value" : 1,
       }, 
       "secondApp": {
           "value":10,
       },
       "thirdApp": {
           "value" : 100,
       }
   },
   {
       "timestamp" : "2019-06-10 11:52",
       "firstApp" : {
           "value" : 10
       }, 
       "secondApp": {
           "value":20,
       },
       "thirdApp": {
           "value" : 115,
       }
   }
]

I have written the following code as such but its taking a lot of time (about 10-12 seconds) (which is quite expected) Please refer to this CodeSandbox (https://codesandbox.io/s/objective-bartik-0r30j?autoresize=1&expanddevtools=1&fontsize=14&hidenavigation=1&module=%2Fsrc%2Findex.js) link to see the code in action.

The problem that I am facing is that it takes a lot of time (and rightfully so) give that

  1. The dataset is huge
  2. My code is bad - since there are too many loops here and there

I have no control over 1 but I have definitely control over 2 . Can you please give me some ideas as to how I can resolve this issue ?

Update Here is a screenshot from the performance tab [![enter image description here][1]][1]

Thanks

let data = {
  response: [
    {
      data: [
        {
          value: 1,
          minute: "2019-06-10 11:51",
          action: "firstApp"
        },
        {
          value: 10,
          minute: "2019-06-10 11:51",
          action: "secondApp"
        },
        {
          value: 100,
          minute: "2019-06-10 11:51",
          action: "thirdApp"
        },
        {
          value: 10,
          minute: "2019-06-10 11:52",
          action: "firstApp"
        },
        {
          value: 20,
          minute: "2019-06-10 11:52",
          action: "secondApp"
        },
        {
          value: 115,
          minute: "2019-06-10 11:52",
          action: "thirdApp"
        }
      ]
    }
  ]
};

function massageData(data) {
  let historyData = [];
  let uniqueTimeStamps = [];
  let event = {
    timestamp: "",
    firstApp: {
      value: 0
    },
    secondApp: {
      value: 0
    },
    thirdApp: {
      value: 0
    }
  };

  for (var i = 0; i < data.length; i++) {
    let item = data[i];

    if (item.minute) {
      if (!uniqueTimeStamps.includes(item.minute)) {
        let timestamp = item.minute;
        console.log("--------------------");
        console.log(timestamp);
        event.timestamp = timestamp;
        event.firstApp.value = getDataValue(data, timestamp, "firstApp");
        event.secondApp.value = getDataValue(data, timestamp, "secondApp");
        event.thirdApp.value = getDataValue(data, timestamp, "thirdApp");
        console.log(event);
        historyData.push(event);
        uniqueTimeStamps.push(item.minute);
      }
    }
  }

  return historyData;
}

function getDataValue(data, timestamp, action) {
  for (var i = 0, len = data.length; i < len; i++) {
    let item = data[i];
    console.log(item);
    if (item["minute"] === timestamp && item["action"] === action) {
      return parseInt(item["value"]);
    }
  }
}

let workData = data.response[0].data;
let formattedData = massageData(workData);
console.log(formattedData);

javascript performance
2 Answers

You could take a classic hash table for same timestamp and update the values with the hash table.

function massageData(data) {
    var historyData = [],
        hash = Object.create(null),
        i, item;

    for (i = 0; i < data.length; i++) {
        item = data[i];
        if (!hash[item.minute]) {
            historyData.push(hash[item.minute] = { timestamp: item.minute, firstApp: { value: 0 }, secondApp: { value: 0 }, thirdApp: { value: 0 } });
        }
        hash[item.minute][item.action].value = item.value;
    }
    return historyData;
}

var data = { response: [{ data: [{ value: 1, minute: "2019-06-10 11:51", action: "firstApp" }, { value: 10, minute: "2019-06-10 11:51", action: "secondApp" }, { value: 100, minute: "2019-06-10 11:51", action: "thirdApp" }, { value: 10, minute: "2019-06-10 11:52", action: "firstApp" }, { value: 20, minute: "2019-06-10 11:52", action: "secondApp" }, { value: 115, minute: "2019-06-10 11:52", action: "thirdApp" }] }] },
    workData = data.response[0].data,
    formattedData = massageData(workData);

console.log(formattedData);
.as-console-wrapper { max-height: 100% !important; top: 0; }

user_14476750

Another similar approach involving the use of an object as a hash table is below. It should be a bit easier to follow:

function massageData (data) {
  const events = [];
  const timestamps = {};

  for (let i = 0; i < data.length; i++) {
    const { action, minute, ...item } = data[i];

    if (!(minute in timestamps)) {
      const event = { timestamp: minute };

      events.push(event);
      timestamps[minute] = event;
    }

    timestamps[minute][action] = item;
  }

  return events;
}

const data = [{ value: 1, minute: "2019-06-10 11:51", action: "firstApp" }, { value: 10, minute: "2019-06-10 11:51", action: "secondApp" }, { value: 100, minute: "2019-06-10 11:51", action: "thirdApp" }, { value: 10, minute: "2019-06-10 11:52", action: "firstApp" }, { value: 20, minute: "2019-06-10 11:52", action: "secondApp" }, { value: 115, minute: "2019-06-10 11:52", action: "thirdApp" }];

console.log(massageData(data));

This will copy all properties in each item except for minute and action into an object in each event as the computed property [action]. If value is really the only other property that exists on each item, then the code can be simplified:

function massageData (data) {
  const events = [];
  const timestamps = {};

  for (let i = 0; i < data.length; i++) {
    const { action, minute, value } = data[i];

    if (!(minute in timestamps)) {
      const event = { timestamp: minute };

      events.push(event);
      timestamps[minute] = event;
    }

    timestamps[minute][action] = { value };
  }

  return events;
}

const data = [{ value: 1, minute: "2019-06-10 11:51", action: "firstApp" }, { value: 10, minute: "2019-06-10 11:51", action: "secondApp" }, { value: 100, minute: "2019-06-10 11:51", action: "thirdApp" }, { value: 10, minute: "2019-06-10 11:52", action: "firstApp" }, { value: 20, minute: "2019-06-10 11:52", action: "secondApp" }, { value: 115, minute: "2019-06-10 11:52", action: "thirdApp" }];

console.log(massageData(data));

user_15415630
Related