Fetch Failing with Apostrophe/Tick Mark Character ' in Node.js

user_1824363 0
Viewed 290

I am writing a Next.js api page that parse the data from a series of URLs. All of the URLs that contain the tick character ' do not return data while the URLs that without it do return data. When I run the same code in Chrome's dev console or a RESTful Client, the URLs with the tick character ' successfully return data. Both scenarios successfully connect (HTTP 200), but results are not found when running in Next.js project. Does anyone know what is causing this difference in behavior?

UPDATES:

  1. I have tried both encodeURI and encodeURIComponent. They do not fix the issue.
  2. I have also tried using AYR%27KA. Does not fix issue.
  3. When I compare the URL passed to fetch to the response.url, the values match in the browser but in Next.js AYR'KA changes to AYR%27KA.
  4. After working with @Phil and doing a lot of digging, this issue is caused by Node.js's URL default component. Here is the NPM link and here is the link to their github. When node-fetch builds a Request object, they call URL.parse on the passed string. Here is a link to URL.parse in the file "url.js" on their github. On lines 353-360, they escape all url component characters from an array named autoEscape. On line 70 where autoEscape, we can see that the apostrophe/tick mark character ' is included in the list of always escaped characters with a comment acknowledging that this is being done despite it not being a restrict character. (NOTE: This update was made on July 24th 2021 and references line numbers and variable names of the latest version of the code at this time.)

*NOTE: To prevent CORS, run in Browser Dev Tools from the site here

const worksEveryWhereURL = "https://robertsspaceindustries.com/api/starmap/star-systems/STANTON",
    brokenInNodeURL = "https://robertsspaceindustries.com/api/starmap/star-systems/AYR'KA";

async function getJSONFrom(url){
    const opts = {method: "POST"};
    let response, json;
    
    console.log("url: ",url)
    response = await fetch( url, {method:"POST"} );
    console.log("response url: ",response.url)
    json = response.json();

    return json;
}

function testResponse(json) {
    try {
        return json.data.resultset.length > 0;
    } catch(err) {
        return false;
    }
}

const example1 = await getJSONFrom( worksEveryWhereURL );
const example2 = await getJSONFrom( brokenInNodeURL );

console.log( testResponse(example1) ); //"true" in chrome and "true" in next.js api page
console.log( testResponse(example2) ); //"true" in chrome and "false" in next.js api page
//Output in browser console when run in browser:
//url: https://robertsspaceindustries.com/api/starmap/star-systems/STANTON
//response url: https://robertsspaceindustries.com/api/starmap/star-systems/STANTON
//url: https://robertsspaceindustries.com/api/starmap/star-systems/AYR'KA
//response url: https://robertsspaceindustries.com/api/starmap/star-systems/AYR'KA <-- matches before fetch
//true
//true <-- expected result

//Output in Command Prompt window when run in Next api page:
//url:  https://robertsspaceindustries.com/api/starmap/star-systems/STANTON
//response url: https://robertsspaceindustries.com/api/starmap/star-systems/STANTON
//url:  https://robertsspaceindustries.com/api/starmap/star-systems/AYR'KA
//response url: https://robertsspaceindustries.com/api/starmap/star-systems/AYR%27KA <-- why changed?
//true
//false <-- unexpect result, note change

Just for reference, here is the code that I am using as an api page that is returning an empty resultset:

export default async function handler(req, res) {
    const worksEveryWhereURL = "https://robertsspaceindustries.com/api/starmap/star-systems/STANTON",
        brokenInNodeURL = "https://robertsspaceindustries.com/api/starmap/star-systems/AYR'KA";

    async function getJSONFrom(url){
        const opts = {method: "POST"};
        let response, json;

        response = await fetch( url, {method:"POST"} );
        json = response.json();

        return json;
    }

    function testResponse(json) {
        try {
            return json.data.resultset.length > 0;
        } catch(err) {
            return false;
        }
    }

    const example1 = await getJSONFrom( worksEveryWhereURL );
    const example2 = await getJSONFrom( brokenInNodeURL );

    console.log( testResponse(example1) ); //"true" in chrome and "true" in next.js api page
    console.log( testResponse(example2) ); //"true" in chrome and "false" in next.js api page
}
//Output in Command Prompt window when run in Next api page:
//url:  https://robertsspaceindustries.com/api/starmap/star-systems/STANTON
//response url: https://robertsspaceindustries.com/api/starmap/star-systems/STANTON
//url:  https://robertsspaceindustries.com/api/starmap/star-systems/AYR'KA
//response url: https://robertsspaceindustries.com/api/starmap/star-systems/AYR%27KA <-- why changed?
//true
//false <-- unexpect result, note change

When the response returns incorrectly, it looks like this:

{
  "success": 1,
  "code": "OK",
  "msg": "OK",
  "data": {
    "rowcount": 0,
    "totalrows": 0,
    "estimatedrows": false,
    "pagesize": 0,
    "pagecount": null,
    "page": 1,
    "offset": 0,
    "startrow": 0,
    "resultset": []
  }
}

While a correct response looks like:

{
  "success": 1,
  "code": "OK",
  "msg": "OK",
  "data": {
    "rowcount": 1,
    "totalrows": 1,
    "estimatedrows": false,
    "pagesize": 0,
    "pagecount": null,
    "page": 1,
    "offset": 0,
    "startrow": 0,
    "resultset": [
      {
        "id": "398",
        "code": "AYR'KA",
        "description": "Primarily a military system focused on supporting, housing, and training Xi’an ground troops. Its close proximity to the Perry Line made it strategically important during the Human / Xi’an cold war. Once relations normalized, Xi’an forces from the Perry Line systems withdrew to here.  ",
        "frost_line": "179.10000000",
        ...
      }
    ]
  }
}

From Update 4: Code snippets from url.js on github that are causing my target URL to be unreachable (not my code; code from a dependency's dependency).

-Lines 56-70 including their comments:

/*
 * RFC 2396: characters reserved for delimiting URLs.
 * We actually just auto-escape these.
 */
delims = [
  '<', '>', '"', '`', ' ', '\r', '\n', '\t'
],

// RFC 2396: characters not allowed for various reasons.
unwise = [
  '{', '}', '|', '\\', '^', '`'
].concat(delims),

// Allowed by RFCs, but cause of XSS attacks.  Always escape these.
autoEscape = ['\''].concat(unwise),

-And here are lines 348-361 including comments were the previous list of characters are used:

/*
 * First, make 100% sure that any "autoEscape" chars get
 * escaped, even if encodeURIComponent doesn't think they
 * need to be.
 */
for (var i = 0, l = autoEscape.length; i < l; i++) {
  var ae = autoEscape[i];
  if (rest.indexOf(ae) === -1) { continue; }
  var esc = encodeURIComponent(ae);
  if (esc === ae) {
    esc = escape(ae);
  }
  rest = rest.split(ae).join(esc);
}
next.js node.js fetch javascript
1 Answers

This seems to be down to the version of node-fetch that's used by Next.js.

The newer v3.x version (still in beta) use new URL() instead of Node's url.parse() which is what's causing the problem.

You might be able to work around this by using new URL() yourself which should avoid any parsing.

Try something like this

async function getJSONFrom(url) {
  const response = await fetch(new URL(url), { method: "POST" })
  if (!response.ok) {
    throw new Error(`${response.status}: ${await response.text()}`)
  }

  return response.json()
}

If that doesn't work (which after looking at the code I'm thinking it might not), try forcing node-fetch@v3.0.0-beta.10

In package.json

"resolutions": {
  "next/node-fetch": "3.0.0-beta.10"
}

See https://classic.yarnpkg.com/en/docs/selective-version-resolutions/

user_2833660
Related