xpath in apps script?

Here is an alternative solution I actually do in a case like this.

I have used XmlService but only for parsing the content, not for using Xpath. This makes use of the element tags and so far pretty consistent on my tests. Although, it might need tweaks when certain tags are in the result and you might have to include them into the exclusion condition.

Tested the code below in both links:

• https://en.wikipedia.org/wiki/Abbeville,_Alabama#Geography
• https://en.wikipedia.org/wiki/Montgomery,_Alabama#Education

My test shows that the formula above used did not return the proper output from the 2nd link while the code does. (Maybe because it was too long)

Code:

function getGeoAndEdu(path) {
  var data = UrlFetchApp.fetch(path).getContentText();
  // wikipedia is divided into sections, if output is cut, increase the number
  var regex = /.{1,100000}/g;
  var results = [];
  // flag to determine if matches should be added
  var foundFlag = false;

  do {
    m = regex.exec(data);
    if (foundFlag) {
      // if another header is found during generation of data, stop appending the matches
      if (matchTag(m[0], "<h2>"))
        foundFlag = false;
      // exclude tables, sub-headers and divs containing image description
      else if(matchTag(m[0], "<div") || matchTag(m[0], "<h3") ||
              matchTag(m[0], "<td")  || matchTag(m[0], "<th"))
        continue;
      else
        results.push(m[0]);
    }
    // start capturing if either IDs are found
    if (m != null && (matchTag(m[0], "id=\"Geography\"") || 
                      matchTag(m[0], "id=\"Education\""))) {
      foundFlag = true;
    }
  } while (m);

  var output = results.map(function (str) {
    // clean tags for XmlService
    str = str.replace(/<[^>]*>/g, '').trim();
    decode = XmlService.parse('<d>' + str + '</d>')
    // convert html entity codes (e.g. &#160;) to text
    return decode.getRootElement().getText();
    // filter blank results due to cleaning and empty sections
    // separate data and remove citations before returning output
  }).filter(result => result.trim().length > 1).join("\n").replace(/\[\d+\]/g, ''); 

  return output;
}

// check if tag is found in string
function matchTag(string, tag) {
  var regex = RegExp(tag);
  return string.match(regex) && string.match(regex)[0] == tag;
}

Output:

Difference:

• Formula ending output
• Script ending output
• Education ending in wikipedia

Note:

• You still have quota when using UrlFetchApp but should be better than IMPORTXML's limit depending on the type of your account.

Reference:

• Apps Script Quotas

Sorry I got very busy this week so I didn't reply. I took a look at your answer which seems to work fine, but it was quite code heavy. I wanted something I would understand so I coded my own solution. not that mine is any simpler. It's just my own code so it's easier for me to follow:

    function getTextBetweenTags(html, paramatersInFirstTag, paramatersInLastTag) {  //finds text values between 2 tags and removes internal tags to leave plain text.
  //eg getTextBetweenTags(html,[['class="mw-headline"'],['id="Geography"']],[['class="wikitable mw-collapsible mw-made-collapsible"']])

  // **Note: you may want to replace &#number; with ascII number

  var openingTagPos = null;
  var closingTagPos = null;
  var previousChar = '';
  var readingTag = false;
  var newTag = '';
  var tagEnd = false;
  var regexFirstTagParams = [];
  var regexLastTagParams = [];

  //prepare regexes to test for parameters in opening and closing tags. put regexes in arrays so each condition can be tested separately
  for (var i in paramatersInFirstTag) {
    regexFirstTagParams.push(new RegExp(escapeRegex(paramatersInFirstTag[i][0])))
  }
  for (var i in paramatersInLastTag) {
    regexLastTagParams.push(new RegExp(escapeRegex(paramatersInLastTag[i][0])))
  }
  var startTagIndex = null;
  var endTagIndex = null;
  var matches = 0;
  for (var i = 0; i < html.length - 1; i++) {
    var nextChar = html.substr(i, 1);

    if (nextChar == '<' && previousChar != '\\') {
      readingTag = true;
    }
    if (nextChar == '>' && previousChar != '\\') {  //if end of tag found, check tag matches start or end tag
      readingTag = false;
      newTag += nextChar;

      //test for firstTag
      if (startTagIndex == null) {
        var alltestsPass = true;
        for (var j in regexFirstTagParams) {
          if (!regexFirstTagParams[j].test(newTag)) alltestsPass = false;
        }
        if (alltestsPass) {
          startTagIndex = i + 1;
          //console.log('Start Tag',startTagIndex)
          matches++;
        }
      }

      //test for lastTag
      else if (startTagIndex != null) {
        var alltestsPass = true;
        for (var j in regexLastTagParams) {
          if (!regexLastTagParams[j].test(newTag)) alltestsPass = false;
        }
        if (alltestsPass) {
          endTagIndex = i + 1;
          matches++;
        }
      }
      if(startTagIndex && endTagIndex) break;
      newTag = '';

    }
    if (readingTag) newTag += nextChar;
    previousChar = nextChar;
  }
  if (matches < 2) return 'No matches';
  else return html.substring(startTagIndex, endTagIndex).replace(/<[^>]+>/g, '');
}

function escapeRegex(string) {
  if (string == null) return string;
  return string.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
}

My function requires an array of attributes for the start tag and an array of attributes for the end tag. It gets any text in between and removes any tags found inbetween. One issue I also noticed was there were often special characters (eg  ) so they need to be replaced. I did that outside the scope of the function above.

The function could be easily improved to check the tag type (eg h2), but it wasn't necessary for the wikipedia case.

Here is a function where I called the above function. the html variable is just the result of UrlFetchApp.fetch('some wikipedia city url').getContextText();

 function getWikiTexts(html) {
      var geography = getTextBetweenTags(html, [['class="mw-headline"'], ['id="Geography']], [['class="mw-headline"']]);
      var economy = getTextBetweenTags(html, 'span', [['class="mw-headline"'], ['id="Economy']], 'span', [['class="mw-headline"']])  
      var education = getTextBetweenTags(html, 'span', [['class="mw-headline"'], ['id="Education']], 'span', [['class="mw-headline"']])
    
      var returnString = '';
      if (geography != 'No matches' && !/Wikipedia/.test(geography)) returnString += geography + '\n';
      if (economy != 'No matches' && !/Wikipedia/.test(economy)) returnString += economy + '\n';
      if (education != 'No matches' && !/Wikipedia/.test(education)) returnString += education + '\n';
    
      return returnString
    }

Thanks for posting your answer.