this.files.forEach(function (f) {
if (f.src.length > 1) {
grunt.log.warn('Only a single src per dest is supported');
return false;
}
var src = f.src.filter(function (filepath) {
// Warn on and remove invalid source files (if nonull was set).
if (!grunt.file.exists(filepath)) {
grunt.log.warn('Source file "' + filepath + '" not found.');
return false;
} else {
return true;
}
}).map(function (filepath) {
// Read file source.
return grunt.file.read(filepath);
})[0],
domParser = new xmldom.DOMParser(),
doc = domParser.parseFromString(src),
xmlSerializer = new xmldom.XMLSerializer(),
replacements = options.replacements || [options];
replacements.forEach(function (replacement) {
var queries = typeof replacement.xpath === 'string' ? [replacement.xpath] : replacement.xpath,
getValue = _.isFunction(replacement.value) ? replacement.value : function () { return replacement.value || ''; },
valueType = typeof replacement.valueType === 'string' ? replacement.valueType : 'text';
queries.forEach(function (query) {
var select = options.namespaces ? xpath.useNamespaces(options.namespaces) : xpath.select;
var nodes = select(query, doc);
nodes.forEach(function (node) {
var value = getValue(node);
grunt.verbose.writeln('setting value of "' + query + '" to "' + value + '"');
if (valueType === 'element') {
node.textContent = '';
while (node.firstChild) {
node.removeChild(node.firstChild);
}
node.appendChild(domParser.parseFromString(value));
}
else if (valueType === 'remove') {
var parentNode = node.parentNode;
parentNode.removeChild(node);
}
else if (node.nodeType === ATTRIBUTE_NODE) {
node.value = value;
} else {
node.textContent = value;
}
});
});
});
// Write the destination file.
grunt.file.write(f.dest, xmlSerializer.serializeToString(doc));
// Print a success message.
grunt.log.writeln('File ' + f.dest.cyan + ' created.');
});
exports.getNodeText = function getNodeText(node) {
if (!node) { return ''; }
var serializer = new xmldom.XMLSerializer(),
str = '';
for (var c = 0; c < node.childNodes.length; c++) {
if (node.childNodes[c].nodeType === 3) {
str += serializer.serializeToString(node.childNodes[c]);
}
}
return str.replace(/\&/g,'&');
};
/**
* Obtain the contents of the block element and prepare it for use in new Template instances
* @name contents
* @type function
* @access internal
* @param DOMElement node
* @return string content
*/
function contents(node)
{
var result = '',
serializer = new XML.XMLSerializer(),
i;
for (i = 0; i < node.childNodes.length; ++i)
result += serializer.serializeToString(node.childNodes[i]);
return result;
}
MeetupsAPI.Bbb.getDefaultConfigXML(ctx, function(err, result) {
if(err || result.returncode != 'success') {
return callback({'code': 503, 'msg': 'Fatal error'});
}
defaultConfigXML = result.defaultConfigXML;
var serializer = new XMLSerializer();
var doc = new DOMParser().parseFromString(defaultConfigXML);
var select = xpath.useNamespaces();
var node;
//// set layout bbb.layout.name.videochat and others
node = select('//layout ', doc, true);
node.setAttribute('defaultLayout', 'bbb.layout.name.videochat');
node.setAttribute('showLayoutTools', 'false');
node.setAttribute('confirmLogout', 'false');
node.setAttribute('showRecordingNotification', 'false');
//// process modules
////// remove desktop sharing
node = xpath.select1("//modules/module[@name=\'DeskShareModule\']", doc);
node.setAttribute('showButton', 'false');
//// remove PhoneModule button
node = xpath.select1("//modules/module[@name=\'PhoneModule\']", doc);
node.setAttribute('showButton', 'true');
node.setAttribute('skipCheck', 'true');
node.setAttribute('listenOnlyMode', 'false');
//// remove VideoconfModule button
node = xpath.select1("//modules/module[@name=\'VideoconfModule\']", doc);
node.setAttribute('showButton', 'true');
node.setAttribute('autoStart', 'true');
node.setAttribute('skipCamSettingsCheck', 'true');
//// remove layout menu
node = xpath.select1("//modules/module[@name=\'LayoutModule\']", doc);
node.setAttribute('enableEdit', 'false');
var xml = serializer.serializeToString(doc);
MeetupsAPI.Bbb.joinURL(ctx, profile, xml, function(err, joinInfo) {
if(err) {
res.send(503, 'Fatal error');
}
MeetupsAPI.emit(MeetupsConstants.events.JOIN_MEETUP, ctx, groupProfile, function(errs) {
});
return callback(null, joinInfo);
});
});
/**
* Functions performed by HTMLReady
*
* State reporting
* - hashtags: collect all #tags in content
* - usertags: collect all @mentions in content
* - htmltags: collect all html <tags> used (for validation)
* - images: collect all image URLs in content
* - links: collect all href URLs in content
*
* Mutations
* - link()
* - ensure all <a> href's begin with a protocol. prepend https:// otherwise.
* - iframe()
* - wrap all <iframe>s in <div class="videoWrapper"> for responsive sizing
* - img()
* - convert any <img> src IPFS prefixes to standard URL
* - change relative protocol to https://
* - linkifyNode()
* - scans text content to be turned into rich content
* - embedYouTubeNode()
* - identify plain youtube URLs and prep them for "rich embed"
* - linkify()
* - scan text for:
* - #tags, convert to <a> links
* - @mentions, convert to <a> links
* - naked URLs
* - if img URL, normalize URL and convert to <img> tag
* - otherwise, normalize URL and convert to <a> link
* - proxifyImages()
* - prepend proxy URL to any non-local <img> src's
*
* We could implement 2 levels of HTML mutation for maximum reuse:
* 1. Normalization of HTML - non-proprietary, pre-rendering cleanup/normalization
* - (state reporting done at this level)
* - normalize URL protocols
* - convert naked URLs to images/links
* - convert embeddable URLs to <iframe>s
* - basic sanitization?
* 2. Steemit.com Rendering - add in proprietary Steemit.com functions/links
* - convert <iframe>s to custom objects
* - linkify #tags and @mentions
* - proxify images
*
* TODO:
* - change ipfsPrefix(url) to normalizeUrl(url)
* - rewrite IPFS prefixes to valid URLs
* - schema normalization
* - gracefully handle protocols like ftp, mailto
*/
/** Split the HTML on top-level elements. This allows react to compare separately, preventing excessive re-rendering.
* Used in MarkdownViewer.jsx
*/
// export function sectionHtml (html) {
// const doc = DOMParser.parseFromString(html, 'text/html')
// const sections = Array(...doc.childNodes).map(child => XMLSerializer.serializeToString(child))
// return sections
// }
/** Embed videos, link mentions and hashtags, etc...
If hideImages and mutate is set to true all images will be replaced
by <pre> elements containing just the image url.
*/
export default function (html, {mutate = true, hideImages = false} = {}) {
const state = {mutate}
state.hashtags = new Set()
state.usertags = new Set()
state.htmltags = new Set()
state.images = new Set()
state.links = new Set()
try {
const doc = DOMParser.parseFromString(html, 'text/html')
traverse(doc, state)
if(mutate) {
if (hideImages) {
for (const image of Array.from(doc.getElementsByTagName('img'))) {
const pre = doc.createElement('pre')
pre.setAttribute('class', 'image-url-only')
pre.appendChild(doc.createTextNode(image.getAttribute('src')))
image.parentNode.replaceChild(pre, image)
}
} else {
proxifyImages(doc)
}
}
// console.log('state', state)
if(!mutate) return state
return {html: (doc) ? XMLSerializer.serializeToString(doc) : '', ...state}
}catch(error) {
// Not Used, parseFromString might throw an error in the future
console.error(error.toString())
return {html}
}
}
this.files.forEach(function (f) {
if (f.src.length > 1) {
grunt.log.warn('Only a single src per dest is supported');
return false;
}
var src = f.src.filter(function (filepath) {
// Warn on and remove invalid source files (if nonull was set).
if (!grunt.file.exists(filepath)) {
grunt.log.warn('Source file "' + filepath + '" not found.');
return false;
} else {
return true;
}
}).map(function (filepath) {
// Read file source.
return grunt.file.read(filepath);
})[0],
domParser = new xmldom.DOMParser(),
doc = domParser.parseFromString(src),
xmlSerializer = new xmldom.XMLSerializer(),
replacements = options.replacements || [options];
replacements.forEach(function (replacement) {
var queries = typeof replacement.xpath === 'string' ? [replacement.xpath] : replacement.xpath,
value = replacement.value || '';
queries.forEach(function (query) {
grunt.verbose.writeln('setting value of "' + query + '" to "' + replacement.value + '"');
var nodes = xpath.select(query, doc);
nodes.forEach(function (node) {
if (node.nodeType === ATTRIBUTE_NODE) {
node.value = value;
} else {
node.textContent = value;
}
});
});
});
// Write the destination file.
grunt.file.write(f.dest, xmlSerializer.serializeToString(doc));
// Print a success message.
grunt.log.writeln('File ' + f.dest.cyan + ' created.');
});
function(data, cb) {
var doc = new DOMParser().parseFromString(data, 'text/xml');
// we expect selectors to match only one node
var nodes = xpath.select(selector, doc);
if(nodes.length > 0) {
var textNode = nodes[0].childNodes[0];
textNode.replaceData(0, textNode.data.length, replace_string);
// write the file back out
var serializer = new XMLSerializer();
var xml = serializer.serializeToString(doc);
fs.writeFile(filePath, xml, cb);
}
else {
cb(null);
}
}
return function node2react(node, index) {
var Tag = "" + node.tagName;
var props = attrs2props(node.attributes);
if (node.nodeType === 3) { // is a text node. nodeType == 3 never gets old!
return serializer.serializeToString(node, rootIndexKey + '.' + index);
} else {
var __html = Array.prototype.slice.call(node.childNodes).map(n=> serializer.serializeToString(n)).join('');
return (<Tag {...props} key={rootIndexKey + '.' + index} dangerouslySetInnerHTML={{__html}}/>)
}
}
function serializeDOM(json, element) {
var xml = '<?xml version="1.0"?>\n';
if (typeof element === 'string') {
xml += element;
} else {
xml += XMLSerializer.serializeToString(element);
}
return xml;
}
function categoryProcessor(node, topic, filename, language)
{
// console.log("Processing category node ", DOMPrinter.serializeToString(node));
var c = {depth: 0, pattern: '*', topic: topic, that: '*', template: '', file: filename};
for (var i = 0; i < node.childNodes.length; i++)
{
var m = node.childNodes[i];
var mName = m.nodeName;
if (mName == '#text') {/*skip*/}
else if (mName == "pattern")
{
c.pattern = trimTag(DOMPrinter.serializeToString(m), 'pattern')
.replace(/[\r\n]/g, '').replace(/\s+/g, ' ').trim();
}
else if (mName == "that")
{
c.that = trimTag(DOMPrinter.serializeToString(m), 'that')
.replace(/[\r\n]/g, '').replace(/\s+/g, ' ').trim();
}
else if (mName == "topic")
{
c.topic = trimTag(DOMPrinter.serializeToString(m), 'topic')
.replace(/[\r\n]/g, '').replace(/\s+/g, ' ').trim();
}
else if (mName == "template")
{
// console.log("Found template tag: " + DOMPrinter.serializeToString(m));
c.template = trimTag(DOMPrinter.serializeToString(m), 'template').trim();
}
else
{
console.log("categoryProcessor: unexpected <" + mName + "> in file ", filename);
}
}
if (!c.template)
{
return null;
}
else
{
return c;
}
}
function linkifyNode(child, state) {try{
const {mutate} = state
if(!child.data) return
const data = XMLSerializer.serializeToString(child)
if(/code/i.test(child.parentNode.tagName)) return
if(/a/i.test(child.parentNode.tagName)) return
const content = linkify(data, state.mutate, state.hashtags, state.usertags, state.images, state.links)
if(mutate && content !== data) {
child.parentNode.replaceChild(DOMParser.parseFromString(`<span>${content}</span>`), child)
}
} catch(error) {console.log(error)}}
sign(root) {
let xml = this.serializer.serializeToString(root);
var sig = new SignedXml();
sig.addReference("/*", ["http://www.w3.org/2000/09/xmldsig#enveloped-signature", "http://www.w3.org/2001/10/xml-exc-c14n#"], "http://www.w3.org/2001/04/xmlenc#sha256");
sig.signingKey = this.priv_key;
sig.signatureAlgorithm = "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256";
sig.computeSignature(xml);
return sig.getSignedXml();
}
function(data, cb) {
var doc = new DOMParser().parseFromString(data, 'text/xml');
var select = xpath.select;
if(input.nsmap) {
select = xpath.useNamespaces(input.nsmap);
}
// we expect selectors to match only one node
var node = select(selector, doc);
if(!!node && node.length > 0) {
node[0].value = replace_string;
// write the file back out
var serializer = new XMLSerializer();
var xml = serializer.serializeToString(doc);
fs.writeFile(filePath, xml, cb);
}
else {
cb(null);
}
}
tiapp.upStackSizeForRhino = function() {
var runtime = U.XML.getNodeText(tiapp.getProperty(doc, 'ti.android.runtime'));
if (runtime === 'rhino') {
var stackSize = tiapp.getProperty(doc, 'ti.android.threadstacksize');
if (stackSize !== null) {
if (parseInt(stackSize.nodeValue, 10) < 32768) {
stackSize.nodeValue('32768');
}
} else {
var node = doc.createElement('property');
var text = doc.createTextNode('32768');
node.setAttribute('name', 'ti.android.threadstacksize');
node.setAttribute('type', 'int');
node.appendChild(text);
doc.documentElement.appendChild(node);
}
// serialize the xml and write to tiapp.xml
var serializer = new XMLSerializer();
var newxml = serializer.serializeToString(doc);
fs.writeFileSync(tiappFile, newxml, 'utf8');
}
};
function loadX3DJS(json, path, xml) {
var version = json.X3D["@version"];
var document = DOMImplementation.createDocument(null, "X3D", docType);
var docType = DOMImplementation.createDocumentType("X3D", 'ISO//Web3D//DTD X3D '+version+'//EN" "http://www.web3d.org/specifications/x3d-'+version+'.dtd', null);
var document = DOMImplementation.createDocument(null, "X3D", docType);
document.insertBefore(document.createProcessingInstruction('xml', 'version="1.0" encoding="UTF-8"'), document.doctype);
var element = document.getElementsByTagNameNS(null, "X3D")[0];
element.setAttribute("xmlns:xsd", 'http://www.w3.org/2001/XMLSchema-instance');
// Bring in JSON to DOM/XML conversion -- used to build DOM/XML.
X3DJSONLD.setDocument(document);
ConvertToX3DOM(json, "", element, path);
xml.push(XMLSerializer.serializeToString(element));
}
module.exports.transform = function (content, options) {
var document = domParser.parseFromString(content);
// move css to the top and combine it
var imgNodes = document.getElementsByTagName("img");
if (imgNodes.length > 0) {
for (var i = 0; i < imgNodes.length; i++) {
var node = imgNodes[i];
node.setAttribute("data-original", node.getAttribute("src"));
node.setAttribute("src", "/loader.gif");
}
}
return xmlSerializer.serializeToString(document.documentElement);
};
function linkifyNode(child, state) {try{
const tag = child.parentNode.tagName ? child.parentNode.tagName.toLowerCase() : child.parentNode.tagName
if(tag === 'code') return
if(tag === 'a') return
const {mutate} = state
if(!child.data) return
if(embedYouTubeNode(child, state.links, state.images)) return
if(embedVimeoNode(child, state.links, state.images)) return
const data = XMLSerializer.serializeToString(child)
const content = linkify(data, state.mutate, state.hashtags, state.usertags, state.images, state.links)
if(mutate && content !== data) {
const newChild = DOMParser.parseFromString(`<span>${content}</span>`)
child.parentNode.replaceChild(newChild, child)
return newChild;
}
} catch(error) {console.log(error)}}
function toXml( values, schema ) {
var doc = domImplementation.createDocument();
var root = doc.createElement( "result" );
var keys = _.keys( schema );
values.map( function( value ) {
var row = doc.createElement( "row" );
keys.forEach( function( key ) {
if ( _.has( value, key ) ) {
row.setAttribute( key, value[ key ] );
}
} );
return row;
} )
.forEach( root.appendChild.bind( root ) );
return xmlSerializer.serializeToString( root );
}
/** Embed videos, link mentions and hashtags, etc...
*/
export default function (html, {large = false, mutate = true}) {
const state = {large, mutate}
state.hashtags = new Set()
state.usertags = new Set()
state.images = new Set()
state.links = new Set()
try {
const doc = DOMParser.parseFromString(html, 'text/html')
traverse(doc, state)
if(mutate) proxifyImages(doc)
// console.log('state', state)
if(!mutate) return state
return {html: XMLSerializer.serializeToString(doc), ...state}
}catch(error) {
// Not Used, parseFromString might throw an error in the future
console.error(error.toString())
return {html}
}
}
// wrap iframes in div.videoWrapper to control size/aspect ratio
function iframe(state, child) {
const url = child.getAttribute('src')
if(url) {
const {images, links} = state
const yt = youTubeId(url)
if(yt && images && links) {
links.add(yt.url)
images.add('https://img.youtube.com/vi/' + yt.id + '/0.jpg')
}
}
const {mutate} = state
if(!mutate) return
const tag = child.parentNode.tagName ? child.parentNode.tagName.toLowerCase() : child.parentNode.tagName
if(tag == 'div' && child.parentNode.getAttribute('class') == 'videoWrapper') return;
const html = XMLSerializer.serializeToString(child)
child.parentNode.replaceChild(DOMParser.parseFromString(`<div class="videoWrapper">${html}</div>`), child)
}
fs.readFile(path, 'utf8', function(error, data) {
if (error) {
deferred.reject(error);
} else {
var doc = new XMLDom().parseFromString(data, 'application/xml');
var nodes = doc.getElementsByTagName(node);
if (nodes.length > 0) {
nodes[0].textContent = value;
fs.writeFile(path, serializer.serializeToString(doc), function(error) {
if (error) {
deferred.reject(error);
} else {
deferred.resolve();
}
});
} else {
deferred.reject('\'' + node + '\' not found in xml file.');
}
}
});
function toXml( values, schema ) {
var doc = domImplementation.createDocument();
var keys = _.keys( schema );
var root = values.map( function( obj ) {
return keys.reduce( function( row, key ) {
var value = obj[ key ];
if ( value !== null && value !== undefined ) {
row.setAttribute( key, _.isDate( value ) ? value.toISOString() : value );
}
return row;
}, doc.createElement( "row" ) );
} )
.reduce( function( parent, child ) {
parent.appendChild( child );
return parent;
}, doc.createElement( "result" ) );
return xmlSerializer.serializeToString( root );
}
function install(type, opts) {
type = type || 'module';
opts = opts || {};
var err = 'Project creation failed. Unable to install ' + type + ' "' +
(opts.name || opts.id) + '"';
// read the tiapp.xml file
var collection = doc.documentElement.getElementsByTagName(type + 's');
var found = false;
// Determine if the module or plugin is already installed
if (collection.length > 0) {
var items = collection.item(0).getElementsByTagName(type);
if (items.length > 0) {
for (var c = 0; c < items.length; c++) {
var theItem = items.item(c);
var theItemText = U.XML.getNodeText(theItem);
if (theItemText == opts.id) {
found = true;
break;
}
}
}
}
// install module or plugin
if (!found) {
// create the node to be inserted
var node = doc.createElement(type);
var text = doc.createTextNode(opts.id);
if (opts.platform) {
node.setAttribute('platform',opts.platform);
}
if (opts.version) {
node.setAttribute('version',opts.version);
}
node.appendChild(text);
// add the node into tiapp.xml
var pna = null;
if (collection.length === 0) {
var pn = doc.createElement(type + 's');
doc.documentElement.appendChild(pn);
doc.documentElement.appendChild(doc.createTextNode("\n"));
pna = pn;
} else {
pna = collection.item(0);
}
pna.appendChild(node);
pna.appendChild(doc.createTextNode("\n"));
// serialize the xml and write to tiapp.xml
var serializer = new XMLSerializer();
var newxml = serializer.serializeToString(doc);
fs.writeFileSync(tiappFile, newxml, 'utf8');
logger.info('Installed "' + opts.id + '" ' + type + ' to ' + tiappFile);
}
}
var suitesEnd = function() {
var serializer = new xmlDom.XMLSerializer();
var docAsString = serializer.serializeToString(this.doc);
console.log(docAsString);
};
this.Then(/^the XML should be$/, function (string, callback) {
var doc = new DOMParser().parseFromString(string),
expected = XMLSerializer.serializeToString(doc);
XMLSerializer.serializeToString(new DOMParser().parseFromString(this.xml)).should.equal(expected);
callback();
});
(function () {
'use strict';
var cheerio = require('cheerio');
var fs = require('fs');
var request = require('request');
var xml2js = require('xml2js');
var xmldom = require('xmldom');
var builder = new xml2js.Builder();
var domParser = new xmldom.DOMParser();
var htmlObj = require('../lib/html');
var utils = require('../lib/utils');
var xmlSerializer = new xmldom.XMLSerializer();
/**
* @param {string} str - The text requiring sane newlines.
* @return {string} A line feed at the end and stripped of carriage returns.
*/
exports.newlineFormat = function (str) {
str = str.replace(/\r/g, '') + '\n';
return str;
};
/**
* @param {array} dataArr - Data array.
* @return {string} Sanitized HTML.
*/
exports.dataArrayToJson = function (dataArr) {
var jsonForData = {html: [{}]};
for (var i = 0; i < dataArr.length; i++) {
for (var j in dataArr[i]) {
if (dataArr[i].hasOwnProperty(j)) {
jsonForData.html[0][j] = dataArr[i][j];
}
}
}
return jsonForData;
};
/**
* @param {string} templateDir - Write destination directory.
* @param {string} fileName - Filename.
* @param {string} fileHtml - Mustache file's content.
* @param {string} fileJson - JSON file's content.
* @param {object} res - response object.
*/
exports.filesWrite = function (templateDir, fileName, fileHtml, fileJson, res) {
try {
fs.writeFileSync(templateDir + '/' + fileName + '.mustache', fileHtml);
fs.writeFileSync(templateDir + '/' + fileName + '.json', fileJson);
exports.redirectWithMsg(res, 'success', 'Go+back+to+the+Pattern+Lab+tab+and+refresh+the+browser+to+check+that+your+template+appears+under+the+Scrape+menu.');
return;
}
catch (err) {
utils.error(err);
}
};
/**
* Sanitize scraped HTML.
*
* @param {string} html - raw HTML.
* @return {string} Sanitized HTML.
*/
exports.htmlSanitize = function (html) {
html = html.replace(/<script(.*?)>/g, '<code$1>');
html = html.replace(/<\/script(.*?)>/g, '</code$1>');
html = html.replace(/<textarea(.*?)>/g, '<figure$1>');
html = html.replace(/<\/textarea(.*?)>/g, '</figure$1>');
return html;
};
/**
* Convert HTML to XHTML for conversion to full JSON data object.
*
* @param {string} targetHtml - XHTML.
* @return {string} XHTML.
*/
exports.htmlToXhtml = function (targetHtml) {
var targetParsed = domParser.parseFromString(targetHtml, 'text/html');
var targetXhtml = xmlSerializer.serializeToString(targetParsed);
return targetXhtml;
};
/**
* @param {string} fileName - Filename.
* @return {boolean} True or false.
*/
exports.isFilenameValid = function (fileName) {
return fileName.match(/^[A-Za-z0-9][\w\-\.]*$/) ? true : false;
};
exports.jsonRecurse = function (jsonObj, dataArr, recursionInc, index, prevIndex) {
var obj;
var underscored;
for (var i in jsonObj) {
if (!jsonObj.hasOwnProperty(i)) {
continue;
}
else if (i !== '_' && i !== '$' && typeof jsonObj[i] === 'object') {
recursionInc++;
exports.jsonRecurse(jsonObj[i], dataArr, recursionInc, i, index);
}
else if (i === '_') {
for (var j in jsonObj) {
if (!jsonObj.hasOwnProperty(j)) {
continue;
}
else if (j !== '$') {
continue;
}
underscored = '';
for (var k in jsonObj[j]) {
if (!jsonObj[j].hasOwnProperty(k)) {
continue;
}
else if (k === 'class') {
underscored = jsonObj[j][k].replace(/-/g, '_').replace(/ /g, '_').replace(/[^\w]/g, '') + '_' + recursionInc;
obj = {};
obj[underscored] = jsonObj[i];
dataArr.push(obj);
break;
}
else if (k === 'id') {
underscored = jsonObj[j][k].replace(/-/g, '_').replace(/ /g, '_').replace(/[^\w]/g, '') + '_' + recursionInc;
obj = {};
obj[underscored] = jsonObj[i];
dataArr.push(obj);
// Don't break because we would prefer to use classes.
}
}
}
if (underscored === '') {
if (typeof index !== 'undefined' && typeof prevIndex !== 'undefined') {
underscored = prevIndex + '_' + recursionInc;
obj = {};
obj[underscored] = jsonObj[i];
dataArr.push(obj);
jsonObj[i] = '{{ ' + underscored + ' }}';
}
}
else {
jsonObj[i] = '{{ ' + underscored + ' }}';
}
}
}
return jsonObj;
};
/**
* @param {object} jsonForXhtml - JSON for conversion to Mustache syntax.
* @return {string} XHTML.
*/
exports.jsonToMustache = function (jsonForXhtml) {
var xhtml = builder.buildObject(jsonForXhtml);
// Remove XML declaration.
xhtml = xhtml.replace(/<\?xml(.|\s)*?\?>/g, '');
// Replace html tags with Mustache tags.
xhtml = xhtml.replace('<html>', '{{# html }}').replace('</html>', '{{/ html }}');
// Clean up.
xhtml = xhtml.replace(/^\s*\n/g, '');
return xhtml;
};
exports.redirectWithMsg = function (res, type, msg, target, url) {
if (res) {
target = typeof target === 'string' ? target : '';
url = typeof url === 'string' ? url : '';
res.writeHead(303, {Location: 'html-scraper?' + type + '=' + msg + '&target=' + target + '&url=' + url});
res.end();
}
};
exports.targetHtmlGet = function ($targetEl, targetIndex, $) {
// Iterate through the collection of selected elements. If an index
// is specified, skip until that index is iterated upon.
var $el;
var innerHtml;
var j = 0;
var outerHtml;
var targetFirst;
var targetHtml = '';
$targetEl.each(function (i, el) {
if (targetIndex === '' || parseInt(targetIndex, 10) === i) {
$el = $(el);
// Cheerio hack for getting outerHTML.
innerHtml = $el.html();
outerHtml = $el.html(innerHtml) + '\n';
targetHtml += outerHtml;
if (j === 0) {
targetFirst = outerHtml;
}
j++;
}
});
return {all: targetHtml, first: targetFirst};
};
exports.targetValidate = function (target, res, req) {
// Split at array index, if any.
var targetSplit = target.split('[', 2);
targetSplit[1] = targetSplit[1] ? targetSplit[1] : '';
// Validate that targetSplit[0] is a css selector.
if (!targetSplit[0].match(/^(#|\.)?[a-z][\w#\-\.]*$/i)) {
exports.redirectWithMsg(res, 'error', 'Incorrect+submission.', req.body.target, req.body.url);
return [];
}
// Remove closing bracket from targetSplit[1] and validate it is an integer.
targetSplit[1] = targetSplit[1].substr(0, targetSplit[1].length - 1);
if (!targetSplit[1].match(/\d*/)) {
exports.redirectWithMsg(res, 'error', 'Incorrect+submission.', req.body.target, req.body.url);
return [];
}
return targetSplit;
};
/**
* Get JSON and array from XHTML.
*
* @param {string} targetXhtml - Well formed XHTML.
* @return {object} Prop 1: json, Prop2: array.
*/
exports.xhtmlToJsonAndArray = function (targetXhtml) {
var dataArr;
var jsonForXhtml;
// Convert to JSON.
xml2js.parseString(targetXhtml, function (err, res) {
if (err) {
utils.error(err);
return {};
}
// jsonRecurse builds dataArr.
dataArr = [];
jsonForXhtml = exports.jsonRecurse(res, dataArr, 0);
});
return {json: jsonForXhtml, array: dataArr};
};
exports.main = function (req, res) {
var $;
var dataArr1;
var dataObj;
var dataStr;
var fileHtml;
var fileJson;
var fileName;
var jsonForData;
var jsonForXhtml;
var output;
var target;
var targetBase;
var $targetEl;
var targetFirst;
var targetHtml;
var targetHtmlObj;
var targetIndex;
var targetSplit;
var targetXhtml;
var templateDir;
var xhtml = '';
// HTML scraper action on submission of URL.
if (typeof req.body.url === 'string' && req.body.url.trim() && typeof req.body.target === 'string') {
try {
request(req.body.url, function (error, response, body) {
if (error || response.statusCode !== 200) {
exports.redirectWithMsg(res, 'error', 'Not+getting+a+valid+response+from+that+URL.', req.body.target, req.body.url);
return;
}
$ = cheerio.load(body);
target = req.body.target.trim();
targetSplit = exports.targetValidate(target, res, req);
targetBase = targetSplit[0];
targetIndex = targetSplit[1];
targetHtml = '';
$targetEl = $(targetBase);
if ($targetEl.length) {
targetHtmlObj = exports.targetHtmlGet($targetEl, targetIndex, $);
// Sanitize scraped HTML.
targetHtml = '<html>' + exports.htmlSanitize(targetHtmlObj.all) + '</html>';
targetFirst = '<html>' + exports.htmlSanitize(targetHtmlObj.first) + '</html>';
// Convert HTML to XHTML for conversion to full JSON data object.
targetXhtml = exports.htmlToXhtml(targetHtml);
// Get array from XHTML.
dataArr1 = exports.xhtmlToJsonAndArray(targetXhtml).array;
// Delete html tags.
targetHtml = targetHtml.replace('<html>', '').replace('</html>', '');
// Convert HTML to XHTML for Mustache template.
targetXhtml = exports.htmlToXhtml(targetFirst);
// Get JSON and array from XHTML.
dataObj = exports.xhtmlToJsonAndArray(targetXhtml);
// Get dataArr2 array. We can't use dataArr1 because we need it
// untouched so we can build jsonForData.
dataArr1 = dataObj.array;
// Build XHTML with mustache tags.
jsonForXhtml = dataObj.json;
xhtml = exports.jsonToMustache(jsonForXhtml);
}
// Convert dataArr1 to JSON and stringify for output.
jsonForData = exports.dataArrayToJson(dataArr1);
dataStr = JSON.stringify(jsonForData, null, 2);
output = '';
output += htmlObj.head;
output += '<section>\n';
output += htmlObj.scraperTitle;
output += htmlObj.reviewerPrefix;
// HTML entities.
output += $('<div/>').text(targetHtml).html().replace(/\n/g, '<br>');
output += htmlObj.reviewerSuffix;
output += htmlObj.importerPrefix;
output += xhtml;
output += htmlObj.json;
output += dataStr;
output += htmlObj.importerSuffix;
output += htmlObj.landingBody;
output += '</section>';
output += htmlObj.foot;
output = output.replace('{{ title }}', 'Fepper HTML Scraper');
output = output.replace('{{ class }}', 'scraper');
output = output.replace('{{ url }}', req.body.url);
output = output.replace('{{ target }}', req.body.target);
res.end(output);
});
}
catch (err) {
utils.error(err);
}
}
// HTML importer action on submission of filename.
else if (typeof req.body.filename === 'string' && req.body.filename !== '') {
// Limit filename characters.
if (!exports.isFilenameValid(req.body.filename)) {
exports.redirectWithMsg(res, 'error', 'Please+enter+a+valid+filename!.', req.body.target, req.body.url);
return;
}
else {
fileName = req.body.filename;
}
templateDir = 'patternlab-node/source/_patterns/98-scrape';
fileHtml = exports.newlineFormat(req.body.html);
fileJson = exports.newlineFormat(req.body.json);
exports.filesWrite(templateDir, fileName, fileHtml, fileJson, res);
}
// If no form variables sent, redirect back with GET.
else {
try {
exports.redirectWithMsg(res, 'error', 'Incorrect+submission.', req.body.target, req.body.url);
return;
}
catch (err) {
utils.error(err);
}
}
};
})();
exports.htmlToXhtml = function (targetHtml) {
var targetParsed = domParser.parseFromString(targetHtml, 'text/html');
var targetXhtml = xmlSerializer.serializeToString(targetParsed);
return targetXhtml;
};
var __html = Array.prototype.slice.call(node.childNodes).map(n=> serializer.serializeToString(n)).join('');
return new Promise((resolve, reject) => {
// No point in generating when there are no files
if ( !files.length ) {
return resolve();
}
// Initialize DOM/XML/SVGO
const DOMParser = new xmldom.DOMParser();
const XMLSerializer = new xmldom.XMLSerializer();
const XMLDoc = new xmldom.DOMImplementation().createDocument(null, null, null);
const SVGOptimizer = new svgo(merge(options.output.svgo, {
plugins: [
// Prevent empty var:* attributes from getting removed prematurely
{ removeEmptyAttrs: false },
// Prevent groups from getting optimized prematurely as they may contain var:* attributes
{ moveGroupAttrsToElems: false },
{ collapseGroups: false },
// Prevent titles from getting removed prematurely
{ removeTitle: false }
]
}));
// Create SVG element
const svg = XMLDoc.createElement('svg');
const sizes = {
width: [],
height: []
};
const formatPostfix = (value) => {
if ( typeof value === 'string' ) {
return value;
}
return '';
};
// Add namespaces
svg.setAttribute('xmlns', 'http://www.w3.org/2000/svg');
if ( options.sprite.generate.use ) {
svg.setAttribute('xmlns:xlink', 'http://www.w3.org/1999/xlink');
}
Promise.all(files.map((file) => new Promise((resolve) => {
const prefix = generateSpritePrefix(options.sprite.prefix, file);
const id = `${prefix}${idify(path.basename(file, path.extname(file)))}`;
let sprite = fs.readFileSync(file, 'utf8');
if ( options.output.svgo === false ) {
return resolve({
id: id,
sprite: sprite
});
}
if ( hasVariables(sprite) && !hasVarNamespace(sprite) ) {
sprite = addVarNamespace(sprite);
}
SVGOptimizer.optimize(sprite, {
path: file
}).then((output) => resolve({
prefix: prefix,
file: file,
id: id,
sprite: output.data
}));
}))).then((items) => {
// Add the xmlns:var attribute when variables are found in any sprite
if ( hasVariables(items.map((item) => item.sprite).join('\n')) ) {
svg.setAttribute(`xmlns:${VAR_NAMESPACE}`, VAR_NAMESPACE_VALUE);
}
items.forEach((item) => {
const sprite = DOMParser.parseFromString(item.sprite).documentElement;
// Attributes that should be transfered to output SVG
const attributes = Array.from(sprite.attributes).reduce((attributes, attribute) => {
// Blacklist several attributes as they'll be added/removed while parsing
if ( ['viewbox', 'width', 'height', 'id', 'xmlns'].includes(attribute.name.toLowerCase()) ) {
return attributes;
}
return [...attributes, {
name: attribute.name,
value: attribute.value
}];
}, []);
// Add xmlns:* attributes to root SVG
attributes.forEach((attribute) => {
if ( !attribute.name.toLowerCase().startsWith('xmlns:') ) {
return;
}
svg.setAttribute(attribute.name, attribute.value);
});
// Get sizes
let viewbox = (sprite.getAttribute('viewBox') || sprite.getAttribute('viewbox')).split(' ').map((a) => parseFloat(a));
let width = parseFloat(sprite.getAttribute('width'));
let height = parseFloat(sprite.getAttribute('height'));
if ( viewbox.length !== 4 && ( isNaN(width) || isNaN(height) ) ) {
return reject(`Invalid SVG '${item.file}'; it's lacking both a viewBox and width/height attributes...`);
}
if ( viewbox.length !== 4 ) {
viewbox = [0, 0, width, height];
}
if ( isNaN(width) ) {
width = viewbox[2];
}
if ( isNaN(height) ) {
height = viewbox[3];
}
// Create symbol
if ( options.sprite.generate.symbol ) {
const symbol = XMLDoc.createElement('symbol');
// Attributes
symbol.setAttribute('id', `${item.id}${formatPostfix(options.sprite.generate.symbol)}`);
symbol.setAttribute('viewBox', viewbox.join(' '));
attributes.forEach((attribute) => {
if ( !['preserveaspectratio'].includes(attribute.name.toLowerCase()) ) {
return;
}
symbol.setAttribute(attribute.name, attribute.value);
});
if ( options.sprite.generate.title ) {
// Make sure we don't overwrite the existing title
const hasTitle = (sprite) => {
const titles = Array.from(sprite.childNodes).filter((childNode) => {
return childNode.nodeName.toLowerCase() === 'title';
});
return !!titles.length;
};
// Add title to improve accessibility
if ( !hasTitle(sprite) ) {
const title = XMLDoc.createElement('title');
title.appendChild(XMLDoc.createTextNode(item.id.replace(item.prefix, '')));
symbol.appendChild(title);
}
}
// Clone the original contents of the SVG file into the new symbol
Array.from(sprite.childNodes).forEach((childNode) => {
symbol.appendChild(childNode);
});
svg.appendChild(symbol);
}
if ( options.sprite.generate.use ) {
// Generate <use> elements within spritemap to allow usage within CSS
const use = XMLDoc.createElement('use');
const y = sizes.height.reduce((a, b) => a + b, 0) + (sizes.height.length * options.sprite.gutter);
use.setAttribute('xlink:href', `#${item.id}${formatPostfix(options.sprite.generate.symbol)}`);
use.setAttribute('x', '0');
use.setAttribute('y', y);
use.setAttribute('width', width.toString());
use.setAttribute('height', height.toString());
svg.appendChild(use);
}
if ( options.sprite.generate.view ) {
// Generate <view> elements within spritemap to allow usage within CSS
const view = XMLDoc.createElement('view');
const y = sizes.height.reduce((a, b) => a + b, 0) + (sizes.height.length * options.sprite.gutter);
// Attributes
view.setAttribute('id', `${item.id}${formatPostfix(options.sprite.generate.view)}`);
view.setAttribute('viewBox', `0 ${Math.max(0, y - (options.sprite.gutter / 2))} ${width + (options.sprite.gutter / 2)} ${height + (options.sprite.gutter / 2)}`);
attributes.forEach((attribute) => {
if ( !['preserveaspectratio'].includes(attribute.name.toLowerCase()) ) {
return;
}
view.setAttribute(attribute.name, attribute.value);
});
svg.appendChild(view);
}
// Update sizes
sizes.width.push(width);
sizes.height.push(height);
});
if ( options.output.svg.sizes ) {
// Add width/height to spritemap
svg.setAttribute('width', Math.max.apply(null, sizes.width).toString());
svg.setAttribute('height', (sizes.height.reduce((a, b) => a + b, 0) + ((sizes.height.length - 1) * options.sprite.gutter)).toString());
}
return resolve(XMLSerializer.serializeToString(svg));
});
});
}))).then((items) => {
// Add the xmlns:var attribute when variables are found in any sprite
if ( hasVariables(items.map((item) => item.sprite).join('\n')) ) {
svg.setAttribute(`xmlns:${VAR_NAMESPACE}`, VAR_NAMESPACE_VALUE);
}
items.forEach((item) => {
const sprite = DOMParser.parseFromString(item.sprite).documentElement;
// Attributes that should be transfered to output SVG
const attributes = Array.from(sprite.attributes).reduce((attributes, attribute) => {
// Blacklist several attributes as they'll be added/removed while parsing
if ( ['viewbox', 'width', 'height', 'id', 'xmlns'].includes(attribute.name.toLowerCase()) ) {
return attributes;
}
return [...attributes, {
name: attribute.name,
value: attribute.value
}];
}, []);
// Add xmlns:* attributes to root SVG
attributes.forEach((attribute) => {
if ( !attribute.name.toLowerCase().startsWith('xmlns:') ) {
return;
}
svg.setAttribute(attribute.name, attribute.value);
});
// Get sizes
let viewbox = (sprite.getAttribute('viewBox') || sprite.getAttribute('viewbox')).split(' ').map((a) => parseFloat(a));
let width = parseFloat(sprite.getAttribute('width'));
let height = parseFloat(sprite.getAttribute('height'));
if ( viewbox.length !== 4 && ( isNaN(width) || isNaN(height) ) ) {
return reject(`Invalid SVG '${item.file}'; it's lacking both a viewBox and width/height attributes...`);
}
if ( viewbox.length !== 4 ) {
viewbox = [0, 0, width, height];
}
if ( isNaN(width) ) {
width = viewbox[2];
}
if ( isNaN(height) ) {
height = viewbox[3];
}
// Create symbol
if ( options.sprite.generate.symbol ) {
const symbol = XMLDoc.createElement('symbol');
// Attributes
symbol.setAttribute('id', `${item.id}${formatPostfix(options.sprite.generate.symbol)}`);
symbol.setAttribute('viewBox', viewbox.join(' '));
attributes.forEach((attribute) => {
if ( !['preserveaspectratio'].includes(attribute.name.toLowerCase()) ) {
return;
}
symbol.setAttribute(attribute.name, attribute.value);
});
if ( options.sprite.generate.title ) {
// Make sure we don't overwrite the existing title
const hasTitle = (sprite) => {
const titles = Array.from(sprite.childNodes).filter((childNode) => {
return childNode.nodeName.toLowerCase() === 'title';
});
return !!titles.length;
};
// Add title to improve accessibility
if ( !hasTitle(sprite) ) {
const title = XMLDoc.createElement('title');
title.appendChild(XMLDoc.createTextNode(item.id.replace(item.prefix, '')));
symbol.appendChild(title);
}
}
// Clone the original contents of the SVG file into the new symbol
Array.from(sprite.childNodes).forEach((childNode) => {
symbol.appendChild(childNode);
});
svg.appendChild(symbol);
}
if ( options.sprite.generate.use ) {
// Generate <use> elements within spritemap to allow usage within CSS
const use = XMLDoc.createElement('use');
const y = sizes.height.reduce((a, b) => a + b, 0) + (sizes.height.length * options.sprite.gutter);
use.setAttribute('xlink:href', `#${item.id}${formatPostfix(options.sprite.generate.symbol)}`);
use.setAttribute('x', '0');
use.setAttribute('y', y);
use.setAttribute('width', width.toString());
use.setAttribute('height', height.toString());
svg.appendChild(use);
}
if ( options.sprite.generate.view ) {
// Generate <view> elements within spritemap to allow usage within CSS
const view = XMLDoc.createElement('view');
const y = sizes.height.reduce((a, b) => a + b, 0) + (sizes.height.length * options.sprite.gutter);
// Attributes
view.setAttribute('id', `${item.id}${formatPostfix(options.sprite.generate.view)}`);
view.setAttribute('viewBox', `0 ${Math.max(0, y - (options.sprite.gutter / 2))} ${width + (options.sprite.gutter / 2)} ${height + (options.sprite.gutter / 2)}`);
attributes.forEach((attribute) => {
if ( !['preserveaspectratio'].includes(attribute.name.toLowerCase()) ) {
return;
}
view.setAttribute(attribute.name, attribute.value);
});
svg.appendChild(view);
}
// Update sizes
sizes.width.push(width);
sizes.height.push(height);
});
if ( options.output.svg.sizes ) {
// Add width/height to spritemap
svg.setAttribute('width', Math.max.apply(null, sizes.width).toString());
svg.setAttribute('height', (sizes.height.reduce((a, b) => a + b, 0) + ((sizes.height.length - 1) * options.sprite.gutter)).toString());
}
return resolve(XMLSerializer.serializeToString(svg));
});