Data file with illegal column names, reading header as keys

user_1483288 0
Viewed 238

The code below has been adopted from https://github.com/emeeks/d3_in_action_2, and specifically from Chapter 4, 4_18.html. My Minimum Reproducible Example is also below.

Question: How does D3 handle illegal characters in column headers? I cannot get D3 to read the values given the illegal keys.

The original header for the example data file looks like this:

day,tweets,retweets,favorite

However, my actual data file has illegal characters in the header, namely, an equals symbol, a space character, and a colon, like this. Note that I cannot change the data file. I can read it, but not write it.

day,twe=ets,re tweets,fav:orite

Here is my test data file with illegal characters in the headers:

day,twe=ets,re tweets,fav:orites
1,1,2,5
2,6,11,3
3,3,8,1
4,5,14,6
5,10,29,16
6,4,22,10
7,3,14,1
8,5,18,7
9,1,30,22
10,4,16,15

Here is the test code, adapted as indicated from Elijah Meeks. Note the <---SEE HERE--- line:

<!doctype html>
<html>
  <head>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/d3/4.2.8/d3.min.js" type="text/JavaScript"></script>
  <style>
    path.domain {
      stroke: none;
    }
  </style>
  </head>
  <body>
    <div id="viz">
      <svg style="width:600px;height:600px;" ></svg>
    </div>
    <script>
      d3.csv("tweetdata2.csv", lineChart);
      
      function lineChart(data) {
        console.log(data); 
        xScale = d3.scaleLinear().domain([1,10.5]).range([20,480]);
        yScale = d3.scaleLinear().domain([0,35]).range([480,20]);
        
        xAxis = d3.axisBottom()
          .scale(xScale)
          .tickSize(480)
          .tickValues([1,2,3,4,5,6,7,8,9,10]);
        
        d3.select("svg").append("g").attr("id", "xAxisG").call(xAxis);
        
        yAxis = d3.axisRight()
          .scale(yScale)
          .ticks(10)
          .tickSize(480);
        d3.select("svg").append("g").attr("id", "yAxisG").call(yAxis);
        
        d3.select("svg").selectAll("circle.tweets")
          .data(data)
          .enter()
          .append("circle")
            .attr("class", "tweets")
            .attr("r", 5)
            .attr("cx", d => xScale(d.day))
            .attr("cy", d => yScale(d.tweets)) // <-------SEE HERE-----
            .style("fill", "#fe9a22");
        
//code commented out for brevity
      }
    </script>
  </body>
</html>

I have tried various permutations of an accessor function, about 8 hours worth of work, and have gotten no-where. Here is one example:

d3.csv("myData.csv", rowConverter, callBack);
function rowConverter(d) 
      {
          console.log("in rowConverter");         
                //for (const property in d) {
                //  console.log(typeof(property));
                //    console.log(d.columns);
                //  console.log(`${property}: ${d[property]}`); }
            d.day =  parseInt(d.day),
            d.tweets =  parseInt(d.twe=eets),
            d.retweets =  parseInt(d.re tweets),
            d.favorites =  parseInt(d.fav:orites)
        return d;
      }
function callBack(d)
{
   console.log(d);
   // etc. omitted for brevity
}

How do I handle this? What do I do? Thanks.

d3.js csv validation javascript
2 Answers

There is nothing in the CSV grammar that say such characters are invalid. That said, your CSV can be used as it is:

const csv = `day,twe=ets,re tweets,fav:orites
1,1,2,5
2,6,11,3
3,3,8,1
4,5,14,6
5,10,29,16
6,4,22,10
7,3,14,1
8,5,18,7
9,1,30,22
10,4,16,15`;
const data = d3.csvParse(csv);
console.log(data);
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/5.7.0/d3.min.js"></script>

The problem is not those characters being invalid in the CSV, the problem here is just accessing those properties (as in foo.twe=ets, which won't work). In that case, you can simply use the bracket notation (as in foo["twe=ets"]):

const csv = `day,twe=ets,re tweets,fav:orites
1,1,2,5
2,6,11,3
3,3,8,1
4,5,14,6
5,10,29,16
6,4,22,10
7,3,14,1
8,5,18,7
9,1,30,22
10,4,16,15`;
const data = d3.csvParse(csv);
console.log(data[0]["twe=ets"]);
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/5.7.0/d3.min.js"></script>

user_57689080

To avoid referring to illegal column values, you can just refer to the data points per Object.values(d) and assign them to the column names you want

var url = "https://gist.githubusercontent.com/robinmackenzie/ff787ddb871cef050d7e6279991a0f07/raw/e59d34ef16b86d23b79dba13f8a778f1485d1c90/data.csv";

d3.csv(url, rowConverter, callBack);

function rowConverter(d, i) {
  console.log("Converting row " + i);
  var vals = Object.values(d); // <--- get the data ignoring object keys
  return {
    "days": parseInt(vals[0]), // <--- use the correct column names
    "tweets": parseInt(vals[1]),
    "retweets": parseInt(vals[2]),
    "favorites": parseInt(vals[3])
  }; 
}

function callBack(d) {
  console.log("Data in callback");
  console.log(d);
}
.as-console-wrapper { max-height: 100% !important; top: 0; }
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/4.2.8/d3.min.js"></script>

Maybe you want to remove the hard-coding of column names inside the row conversion function and if so, you can define them ahead of time like below:

var url = "https://gist.githubusercontent.com/robinmackenzie/ff787ddb871cef050d7e6279991a0f07/raw/e59d34ef16b86d23b79dba13f8a778f1485d1c90/data.csv";

var cols = ["days_xxx", "tweets_xxx", "retweets_xxx", "favorites_xxx"];

d3.csv(url, rowConverter, callBack);

function rowConverter(d, i) {
  console.log("Converting row " + i);
  var vals = Object.values(d); // <--- get the data ignoring object keys
  var obj = {}
  for ([ix, v] of vals.entries()) { // <--- get index and value of array of values
    obj[cols[ix]] = +v; // <--- use the index to get the column name from array
  }
  console.log(obj);
  return obj; 
}

function callBack(d) {
  console.log("Data in callback");
  console.log(d);
}
.as-console-wrapper { max-height: 100% !important; top: 0; }
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/4.2.8/d3.min.js"></script>

user_7653950
Related