Example #1
0
        this.files.forEach(function (f) {
            if (f.src.length > 1) {
                grunt.log.warn('Only a single src per dest is supported');
                return false;
            }
            
            var src = f.src.filter(function (filepath) {
                    // Warn on and remove invalid source files (if nonull was set).
                    if (!grunt.file.exists(filepath)) {
                        grunt.log.warn('Source file "' + filepath + '" not found.');
                        return false;
                    } else {
                        return true;
                    }
                }).map(function (filepath) {
                    // Read file source.
                    return grunt.file.read(filepath);
                })[0],
                domParser = new xmldom.DOMParser(),
                doc = domParser.parseFromString(src),
                xmlSerializer = new xmldom.XMLSerializer(),
                replacements = options.replacements || [options];
            
            replacements.forEach(function (replacement) {
                var queries = typeof replacement.xpath === 'string' ? [replacement.xpath] : replacement.xpath,
                    getValue = _.isFunction(replacement.value) ? replacement.value : function () { return replacement.value || ''; },
                    valueType = typeof replacement.valueType === 'string' ? replacement.valueType : 'text';
                queries.forEach(function (query) {
                    var select = options.namespaces ? xpath.useNamespaces(options.namespaces) : xpath.select;
                    var nodes = select(query, doc);
                    nodes.forEach(function (node) {
                        var value = getValue(node);
                        grunt.verbose.writeln('setting value of "' + query + '" to "' + value + '"');
                        if (valueType === 'element') {
                            node.textContent = '';
                            while (node.firstChild) {
                                node.removeChild(node.firstChild);
                            }
                            node.appendChild(domParser.parseFromString(value));
                        }
                        else if (valueType === 'remove') {
                            var parentNode = node.parentNode;
                            parentNode.removeChild(node);
                        }
                        else if (node.nodeType === ATTRIBUTE_NODE) {
                            node.value = value;
                        } else {
                            node.textContent = value;
                        }
                    });
                });
            });

            // Write the destination file.
            grunt.file.write(f.dest, xmlSerializer.serializeToString(doc));

            // Print a success message.
            grunt.log.writeln('File ' + f.dest.cyan + ' created.');
        });
Example #2
0
exports.getNodeText = function getNodeText(node) {
	if (!node) { return ''; }
	var serializer = new xmldom.XMLSerializer(),
		str = '';
	for (var c = 0; c < node.childNodes.length; c++) {
		if (node.childNodes[c].nodeType === 3) {
			str += serializer.serializeToString(node.childNodes[c]);
		}
	}
	return str.replace(/\&amp;/g,'&');
};
Example #3
0
	/**
	 *  Obtain the contents of the block element and prepare it for use in new Template instances
	 *  @name    contents
	 *  @type    function
	 *  @access  internal
	 *  @param   DOMElement node
	 *  @return  string content
	 */
	function contents(node)
	{
		var result = '',
			serializer = new XML.XMLSerializer(),
			i;

		for (i = 0; i < node.childNodes.length; ++i)
			result += serializer.serializeToString(node.childNodes[i]);

		return result;
	}
Example #4
0
        MeetupsAPI.Bbb.getDefaultConfigXML(ctx, function(err, result) {
            if(err || result.returncode != 'success') {
                return callback({'code': 503, 'msg': 'Fatal error'});
            }

            defaultConfigXML = result.defaultConfigXML;
            var serializer = new XMLSerializer();
            var doc = new DOMParser().parseFromString(defaultConfigXML);
            var select = xpath.useNamespaces();
            var node;

            //// set layout bbb.layout.name.videochat and others
            node = select('//layout ', doc, true);
            node.setAttribute('defaultLayout', 'bbb.layout.name.videochat');
            node.setAttribute('showLayoutTools', 'false');
            node.setAttribute('confirmLogout', 'false');
            node.setAttribute('showRecordingNotification', 'false');
            //// process modules
            ////// remove desktop sharing
            node = xpath.select1("//modules/module[@name=\'DeskShareModule\']", doc);
            node.setAttribute('showButton', 'false');
            //// remove PhoneModule button
            node = xpath.select1("//modules/module[@name=\'PhoneModule\']", doc);
            node.setAttribute('showButton', 'true');
            node.setAttribute('skipCheck', 'true');
            node.setAttribute('listenOnlyMode', 'false');
            //// remove VideoconfModule button
            node = xpath.select1("//modules/module[@name=\'VideoconfModule\']", doc);
            node.setAttribute('showButton', 'true');
            node.setAttribute('autoStart', 'true');
            node.setAttribute('skipCamSettingsCheck', 'true');
            //// remove layout menu
            node = xpath.select1("//modules/module[@name=\'LayoutModule\']", doc);
            node.setAttribute('enableEdit', 'false');

            var xml = serializer.serializeToString(doc);
            MeetupsAPI.Bbb.joinURL(ctx, profile, xml, function(err, joinInfo) {
                if(err) {
                    res.send(503, 'Fatal error');
                }

                MeetupsAPI.emit(MeetupsConstants.events.JOIN_MEETUP, ctx, groupProfile, function(errs) {

                });

                return callback(null, joinInfo);
            });
        });
Example #5
0
/**
 * Functions performed by HTMLReady
 *
 * State reporting
 *  - hashtags: collect all #tags in content
 *  - usertags: collect all @mentions in content
 *  - htmltags: collect all html <tags> used (for validation)
 *  - images: collect all image URLs in content
 *  - links: collect all href URLs in content
 *
 * Mutations
 *  - link()
 *    - ensure all <a> href's begin with a protocol. prepend https:// otherwise.
 *  - iframe()
 *    - wrap all <iframe>s in <div class="videoWrapper"> for responsive sizing
 *  - img()
 *    - convert any <img> src IPFS prefixes to standard URL
 *    - change relative protocol to https://
 *  - linkifyNode()
 *    - scans text content to be turned into rich content
 *    - embedYouTubeNode()
 *      - identify plain youtube URLs and prep them for "rich embed"
 *    - linkify()
 *      - scan text for:
 *        - #tags, convert to <a> links
 *        - @mentions, convert to <a> links
 *        - naked URLs
 *          - if img URL, normalize URL and convert to <img> tag
 *          - otherwise, normalize URL and convert to <a> link
 *  - proxifyImages()
 *    - prepend proxy URL to any non-local <img> src's
 *
 * We could implement 2 levels of HTML mutation for maximum reuse:
 *  1. Normalization of HTML - non-proprietary, pre-rendering cleanup/normalization
 *    - (state reporting done at this level)
 *    - normalize URL protocols
 *    - convert naked URLs to images/links
 *    - convert embeddable URLs to <iframe>s
 *    - basic sanitization?
 *  2. Steemit.com Rendering - add in proprietary Steemit.com functions/links
 *    - convert <iframe>s to custom objects
 *    - linkify #tags and @mentions
 *    - proxify images
 *
 * TODO:
 *  - change ipfsPrefix(url) to normalizeUrl(url)
 *    - rewrite IPFS prefixes to valid URLs
 *    - schema normalization
 *    - gracefully handle protocols like ftp, mailto
 */

/** Split the HTML on top-level elements. This allows react to compare separately, preventing excessive re-rendering.
 * Used in MarkdownViewer.jsx
 */
// export function sectionHtml (html) {
//   const doc = DOMParser.parseFromString(html, 'text/html')
//   const sections = Array(...doc.childNodes).map(child => XMLSerializer.serializeToString(child))
//   return sections
// }

/** Embed videos, link mentions and hashtags, etc...
    If hideImages and mutate is set to true all images will be replaced
    by <pre> elements containing just the image url.
*/
export default function (html, {mutate = true, hideImages = false} = {}) {
    const state = {mutate}
    state.hashtags = new Set()
    state.usertags = new Set()
    state.htmltags = new Set()
    state.images = new Set()
    state.links = new Set()
    try {
        const doc = DOMParser.parseFromString(html, 'text/html')
        traverse(doc, state)
        if(mutate) {
            if (hideImages) {
                for (const image of Array.from(doc.getElementsByTagName('img'))) {
                    const pre = doc.createElement('pre')
                    pre.setAttribute('class', 'image-url-only')
                    pre.appendChild(doc.createTextNode(image.getAttribute('src')))
                    image.parentNode.replaceChild(pre, image)
                }
            } else {
                proxifyImages(doc)
            }
        }
        // console.log('state', state)
        if(!mutate) return state
        return {html: (doc) ? XMLSerializer.serializeToString(doc) : '', ...state}
    }catch(error) {
        // Not Used, parseFromString might throw an error in the future
        console.error(error.toString())
        return {html}
    }
}
Example #6
0
        this.files.forEach(function (f) {
            if (f.src.length > 1) {
                grunt.log.warn('Only a single src per dest is supported');
                return false;
            }
            
            var src = f.src.filter(function (filepath) {
                    // Warn on and remove invalid source files (if nonull was set).
                    if (!grunt.file.exists(filepath)) {
                        grunt.log.warn('Source file "' + filepath + '" not found.');
                        return false;
                    } else {
                        return true;
                    }
                }).map(function (filepath) {
                    // Read file source.
                    return grunt.file.read(filepath);
                })[0],
                domParser = new xmldom.DOMParser(),
                doc = domParser.parseFromString(src),
                xmlSerializer = new xmldom.XMLSerializer(),
                replacements = options.replacements || [options];
            
            replacements.forEach(function (replacement) {
                var queries = typeof replacement.xpath === 'string' ? [replacement.xpath] : replacement.xpath,
                    value = replacement.value || '';
                queries.forEach(function (query) {
                    grunt.verbose.writeln('setting value of "' + query + '" to "' + replacement.value + '"');
                    var nodes = xpath.select(query, doc);
                    nodes.forEach(function (node) {
                        if (node.nodeType === ATTRIBUTE_NODE) {
                            node.value = value;
                        } else {
                            node.textContent = value;
                        }
                    });
                });
            });

            // Write the destination file.
            grunt.file.write(f.dest, xmlSerializer.serializeToString(doc));

            // Print a success message.
            grunt.log.writeln('File ' + f.dest.cyan + ' created.');
        });
Example #7
0
         function(data, cb) {
             var doc = new DOMParser().parseFromString(data, 'text/xml');
             
             // we expect selectors to match only one node
             var nodes = xpath.select(selector, doc);
             if(nodes.length > 0) {
                 var textNode = nodes[0].childNodes[0];
                 textNode.replaceData(0, textNode.data.length, replace_string);
 
                 // write the file back out
                 var serializer = new XMLSerializer();
                 var xml = serializer.serializeToString(doc);
                 fs.writeFile(filePath, xml, cb);
             }
             else {
                 cb(null);
             }
         }
 return function node2react(node, index) {
     var Tag = "" + node.tagName;
     var props = attrs2props(node.attributes);
     if (node.nodeType === 3) { // is a text node. nodeType == 3 never gets old!
         return serializer.serializeToString(node, rootIndexKey + '.' + index);
     } else {
         var __html = Array.prototype.slice.call(node.childNodes).map(n=> serializer.serializeToString(n)).join('');
         return (<Tag {...props} key={rootIndexKey + '.' + index} dangerouslySetInnerHTML={{__html}}/>)
     }
 }
Example #9
0
function serializeDOM(json, element) {
	var xml = '<?xml version="1.0"?>\n';
	if (typeof element === 'string') {
		xml += element;
	} else {
		xml += XMLSerializer.serializeToString(element);
	}

	return xml;
}
Example #10
0
function categoryProcessor(node, topic, filename, language)
{
  // console.log("Processing category node ", DOMPrinter.serializeToString(node));
  var c = {depth: 0, pattern: '*', topic: topic, that: '*', template: '', file: filename};
  for (var i = 0; i < node.childNodes.length; i++)
  {
    var m = node.childNodes[i];
    var mName = m.nodeName;
    if (mName == '#text') {/*skip*/}
    else if (mName == "pattern")
    {
      c.pattern = trimTag(DOMPrinter.serializeToString(m), 'pattern')
      .replace(/[\r\n]/g, '').replace(/\s+/g, ' ').trim();
    }
    else if (mName == "that")
    {
      c.that = trimTag(DOMPrinter.serializeToString(m), 'that')
      .replace(/[\r\n]/g, '').replace(/\s+/g, ' ').trim();
    }
    else if (mName == "topic")
    {
      c.topic = trimTag(DOMPrinter.serializeToString(m), 'topic')
      .replace(/[\r\n]/g, '').replace(/\s+/g, ' ').trim();
    }
    else if (mName == "template")
    {
      // console.log("Found template tag: " + DOMPrinter.serializeToString(m));
      c.template = trimTag(DOMPrinter.serializeToString(m), 'template').trim();
    }
    else
    {
      console.log("categoryProcessor: unexpected <" + mName + "> in file ", filename);
    }
  }
  if (!c.template)
  {
    return null;
  }
  else
  {
    return c;
  }
}
Example #11
0
function linkifyNode(child, state) {try{
    const {mutate} = state
    if(!child.data) return
    const data = XMLSerializer.serializeToString(child)
    if(/code/i.test(child.parentNode.tagName)) return
    if(/a/i.test(child.parentNode.tagName)) return
    const content = linkify(data, state.mutate, state.hashtags, state.usertags, state.images, state.links)
    if(mutate && content !== data) {
        child.parentNode.replaceChild(DOMParser.parseFromString(`<span>${content}</span>`), child)
    }
} catch(error) {console.log(error)}}
Example #12
0
	sign(root) {
		let xml = this.serializer.serializeToString(root);

		var sig = new SignedXml();

		sig.addReference("/*", ["http://www.w3.org/2000/09/xmldsig#enveloped-signature", "http://www.w3.org/2001/10/xml-exc-c14n#"], "http://www.w3.org/2001/04/xmlenc#sha256");
		sig.signingKey = this.priv_key;
		sig.signatureAlgorithm = "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256";
		sig.computeSignature(xml);

		return sig.getSignedXml();
	}
Example #13
0
            function(data, cb) {
                var doc = new DOMParser().parseFromString(data, 'text/xml');

                var select = xpath.select;
                if(input.nsmap) {
                    select = xpath.useNamespaces(input.nsmap);
                }

                // we expect selectors to match only one node
                var node = select(selector, doc);
                if(!!node && node.length > 0) {
                    node[0].value = replace_string;

                    // write the file back out
                    var serializer = new XMLSerializer();
                    var xml = serializer.serializeToString(doc);
                    fs.writeFile(filePath, xml, cb);
                }
                else {
                    cb(null);
                }
            }
Example #14
0
tiapp.upStackSizeForRhino = function() {
	var runtime = U.XML.getNodeText(tiapp.getProperty(doc, 'ti.android.runtime'));
	if (runtime === 'rhino') {
		var stackSize = tiapp.getProperty(doc, 'ti.android.threadstacksize');
		if (stackSize !== null) {
			if (parseInt(stackSize.nodeValue, 10) < 32768) {
				stackSize.nodeValue('32768');
			}
		} else {
			var node = doc.createElement('property');
			var text = doc.createTextNode('32768');
			node.setAttribute('name', 'ti.android.threadstacksize');
			node.setAttribute('type', 'int');
			node.appendChild(text);
			doc.documentElement.appendChild(node);
		}

		// serialize the xml and write to tiapp.xml
		var serializer = new XMLSerializer();
		var newxml = serializer.serializeToString(doc);
		fs.writeFileSync(tiappFile, newxml, 'utf8');
	}
};
Example #15
0
function loadX3DJS(json, path, xml) {
	var version = json.X3D["@version"];
	var document = DOMImplementation.createDocument(null, "X3D", docType);
	var docType = DOMImplementation.createDocumentType("X3D", 'ISO//Web3D//DTD X3D '+version+'//EN" "http://www.web3d.org/specifications/x3d-'+version+'.dtd', null);
	var document = DOMImplementation.createDocument(null, "X3D", docType);
	document.insertBefore(document.createProcessingInstruction('xml', 'version="1.0" encoding="UTF-8"'), document.doctype);
	var element = document.getElementsByTagNameNS(null, "X3D")[0];
	element.setAttribute("xmlns:xsd", 'http://www.w3.org/2001/XMLSchema-instance');

	// Bring in JSON to DOM/XML conversion --  used to build DOM/XML.
	X3DJSONLD.setDocument(document);

	ConvertToX3DOM(json, "", element, path);
	xml.push(XMLSerializer.serializeToString(element));
}
Example #16
0
module.exports.transform = function (content, options) {
    var document = domParser.parseFromString(content);

    // move css to the top and combine it
    var imgNodes = document.getElementsByTagName("img");
    if (imgNodes.length > 0) {
        for (var i = 0; i < imgNodes.length; i++) {
            var node = imgNodes[i];


            node.setAttribute("data-original", node.getAttribute("src"));
            node.setAttribute("src", "/loader.gif");
        }
    }

    return xmlSerializer.serializeToString(document.documentElement);
};
Example #17
0
function linkifyNode(child, state) {try{
    const tag = child.parentNode.tagName ? child.parentNode.tagName.toLowerCase() : child.parentNode.tagName
    if(tag === 'code') return
    if(tag === 'a') return

    const {mutate} = state
    if(!child.data) return
    if(embedYouTubeNode(child, state.links, state.images)) return
    if(embedVimeoNode(child, state.links, state.images)) return

    const data = XMLSerializer.serializeToString(child)
    const content = linkify(data, state.mutate, state.hashtags, state.usertags, state.images, state.links)
    if(mutate && content !== data) {
        const newChild = DOMParser.parseFromString(`<span>${content}</span>`)
        child.parentNode.replaceChild(newChild, child)
        return newChild;
    }
} catch(error) {console.log(error)}}
Example #18
0
function toXml( values, schema ) {
	var doc = domImplementation.createDocument();
	var root = doc.createElement( "result" );
	var keys = _.keys( schema );

	values.map( function( value ) {
		var row = doc.createElement( "row" );
		keys.forEach( function( key ) {
			if ( _.has( value, key ) ) {
				row.setAttribute( key, value[ key ] );
			}
		} );
		return row;
	} )
	.forEach( root.appendChild.bind( root ) );

	return xmlSerializer.serializeToString( root );
}
Example #19
0
/** Embed videos, link mentions and hashtags, etc...
*/
export default function (html, {large = false, mutate = true}) {
    const state = {large, mutate}
    state.hashtags = new Set()
    state.usertags = new Set()
    state.images = new Set()
    state.links = new Set()
    try {
        const doc = DOMParser.parseFromString(html, 'text/html')
        traverse(doc, state)
        if(mutate) proxifyImages(doc)
        // console.log('state', state)
        if(!mutate) return state
        return {html: XMLSerializer.serializeToString(doc), ...state}
    }catch(error) {
        // Not Used, parseFromString might throw an error in the future
        console.error(error.toString())
        return {html}
    }
}
Example #20
0
// wrap iframes in div.videoWrapper to control size/aspect ratio
function iframe(state, child) {
    const url = child.getAttribute('src')
    if(url) {
        const {images, links} = state
        const yt = youTubeId(url)
        if(yt && images && links) {
            links.add(yt.url)
            images.add('https://img.youtube.com/vi/' + yt.id + '/0.jpg')
        }
    }

    const {mutate} = state
    if(!mutate) return

    const tag = child.parentNode.tagName ? child.parentNode.tagName.toLowerCase() : child.parentNode.tagName
    if(tag == 'div' && child.parentNode.getAttribute('class') == 'videoWrapper') return;
    const html = XMLSerializer.serializeToString(child)
    child.parentNode.replaceChild(DOMParser.parseFromString(`<div class="videoWrapper">${html}</div>`), child)
}
Example #21
0
 fs.readFile(path, 'utf8', function(error, data) {
   if (error) {
     deferred.reject(error);
   } else {
     var doc = new XMLDom().parseFromString(data, 'application/xml');
     var nodes = doc.getElementsByTagName(node);
     if (nodes.length > 0) {
       nodes[0].textContent = value;
       fs.writeFile(path, serializer.serializeToString(doc), function(error) {
         if (error) {
           deferred.reject(error);
         } else {
           deferred.resolve();
         }
       });
     } else {
       deferred.reject('\'' + node + '\' not found in xml file.');
     }
   }
 });
Example #22
0
function toXml( values, schema ) {
	var doc = domImplementation.createDocument();
	var keys = _.keys( schema );

	var root = values.map( function( obj ) {
		return keys.reduce( function( row, key ) {
			var value = obj[ key ];
			if ( value !== null && value !== undefined ) {
				row.setAttribute( key, _.isDate( value ) ? value.toISOString() : value );
			}
			return row;
		}, doc.createElement( "row" ) );
	} )
	.reduce( function( parent, child ) {
		parent.appendChild( child );
		return parent;
	}, doc.createElement( "result" ) );

	return xmlSerializer.serializeToString( root );
}
Example #23
0
function install(type, opts) {
	type = type || 'module';
	opts = opts || {};

	var err = 'Project creation failed. Unable to install ' + type + ' "' +
		(opts.name || opts.id) + '"';

	// read the tiapp.xml file
	var collection = doc.documentElement.getElementsByTagName(type + 's');
	var found = false;

	// Determine if the module or plugin is already installed
	if (collection.length > 0) {
		var items = collection.item(0).getElementsByTagName(type);
		if (items.length > 0) {
			for (var c = 0; c < items.length; c++) {
				var theItem = items.item(c);
				var theItemText = U.XML.getNodeText(theItem);
				if (theItemText == opts.id) {
					found = true;
					break;
				}
			}
		}
	}

	// install module or plugin
	if (!found) {
		// create the node to be inserted
		var node = doc.createElement(type);
		var text = doc.createTextNode(opts.id);
		if (opts.platform) {
			node.setAttribute('platform',opts.platform);
		}
		if (opts.version) {
			node.setAttribute('version',opts.version);
		}
		node.appendChild(text);

		// add the node into tiapp.xml
		var pna = null;
		if (collection.length === 0) {
			var pn = doc.createElement(type + 's');
			doc.documentElement.appendChild(pn);
			doc.documentElement.appendChild(doc.createTextNode("\n"));
			pna = pn;
		} else {
			pna = collection.item(0);
		}
		pna.appendChild(node);
		pna.appendChild(doc.createTextNode("\n"));

		// serialize the xml and write to tiapp.xml
		var serializer = new XMLSerializer();
		var newxml = serializer.serializeToString(doc);
		fs.writeFileSync(tiappFile, newxml, 'utf8');

		logger.info('Installed "' + opts.id + '" ' + type + ' to ' + tiappFile);
	}

}
Example #24
0
var suitesEnd = function() {
    var serializer = new xmlDom.XMLSerializer();
    var docAsString = serializer.serializeToString(this.doc);
    console.log(docAsString);
};
Example #25
0
 this.Then(/^the XML should be$/, function (string, callback) {
   var doc = new DOMParser().parseFromString(string),
       expected = XMLSerializer.serializeToString(doc);
   XMLSerializer.serializeToString(new DOMParser().parseFromString(this.xml)).should.equal(expected);
   callback();
 });
Example #26
0
(function () {
  'use strict';

  var cheerio = require('cheerio');
  var fs = require('fs');
  var request = require('request');
  var xml2js = require('xml2js');
  var xmldom = require('xmldom');

  var builder = new xml2js.Builder();
  var domParser = new xmldom.DOMParser();
  var htmlObj = require('../lib/html');
  var utils = require('../lib/utils');
  var xmlSerializer = new xmldom.XMLSerializer();

  /**
   * @param {string} str - The text requiring sane newlines.
   * @return {string} A line feed at the end and stripped of carriage returns.
   */
  exports.newlineFormat = function (str) {
    str = str.replace(/\r/g, '') + '\n';

    return str;
  };

  /**
   * @param {array} dataArr - Data array.
   * @return {string} Sanitized HTML.
   */
  exports.dataArrayToJson = function (dataArr) {
    var jsonForData = {html: [{}]};

    for (var i = 0; i < dataArr.length; i++) {
      for (var j in dataArr[i]) {
        if (dataArr[i].hasOwnProperty(j)) {
          jsonForData.html[0][j] = dataArr[i][j];
        }
      }
    }

    return jsonForData;
  };

  /**
   * @param {string} templateDir - Write destination directory.
   * @param {string} fileName - Filename.
   * @param {string} fileHtml - Mustache file's content.
   * @param {string} fileJson - JSON file's content.
   * @param {object} res - response object.
   */
  exports.filesWrite = function (templateDir, fileName, fileHtml, fileJson, res) {
    try {
      fs.writeFileSync(templateDir + '/' + fileName + '.mustache', fileHtml);
      fs.writeFileSync(templateDir + '/' + fileName + '.json', fileJson);

      exports.redirectWithMsg(res, 'success', 'Go+back+to+the+Pattern+Lab+tab+and+refresh+the+browser+to+check+that+your+template+appears+under+the+Scrape+menu.');
      return;
    }
    catch (err) {
      utils.error(err);
    }
  };

  /**
   * Sanitize scraped HTML.
   *
   * @param {string} html - raw HTML.
   * @return {string} Sanitized HTML.
   */
  exports.htmlSanitize = function (html) {
    html = html.replace(/<script(.*?)>/g, '<code$1>');
    html = html.replace(/<\/script(.*?)>/g, '</code$1>');
    html = html.replace(/<textarea(.*?)>/g, '<figure$1>');
    html = html.replace(/<\/textarea(.*?)>/g, '</figure$1>');

    return html;
  };

  /**
   * Convert HTML to XHTML for conversion to full JSON data object.
   *
   * @param {string} targetHtml - XHTML.
   * @return {string} XHTML.
   */
  exports.htmlToXhtml = function (targetHtml) {
    var targetParsed = domParser.parseFromString(targetHtml, 'text/html');
    var targetXhtml = xmlSerializer.serializeToString(targetParsed);

    return targetXhtml;
  };

  /**
   * @param {string} fileName - Filename.
   * @return {boolean} True or false.
   */
  exports.isFilenameValid = function (fileName) {
    return fileName.match(/^[A-Za-z0-9][\w\-\.]*$/) ? true : false;
  };

  exports.jsonRecurse = function (jsonObj, dataArr, recursionInc, index, prevIndex) {
    var obj;
    var underscored;

    for (var i in jsonObj) {
      if (!jsonObj.hasOwnProperty(i)) {
        continue;
      }

      else if (i !== '_' && i !== '$' && typeof jsonObj[i] === 'object') {
        recursionInc++;
        exports.jsonRecurse(jsonObj[i], dataArr, recursionInc, i, index);
      }

      else if (i === '_') {
        for (var j in jsonObj) {
          if (!jsonObj.hasOwnProperty(j)) {
            continue;
          }

          else if (j !== '$') {
            continue;
          }

          underscored = '';
          for (var k in jsonObj[j]) {
            if (!jsonObj[j].hasOwnProperty(k)) {
              continue;
            }

            else if (k === 'class') {
              underscored = jsonObj[j][k].replace(/-/g, '_').replace(/ /g, '_').replace(/[^\w]/g, '') + '_' + recursionInc;
              obj = {};
              obj[underscored] = jsonObj[i];
              dataArr.push(obj);
              break;
            }

            else if (k === 'id') {
              underscored = jsonObj[j][k].replace(/-/g, '_').replace(/ /g, '_').replace(/[^\w]/g, '') + '_' + recursionInc;
              obj = {};
              obj[underscored] = jsonObj[i];
              dataArr.push(obj);
              // Don't break because we would prefer to use classes.
            }
          }
        }

        if (underscored === '') {
          if (typeof index !== 'undefined' && typeof prevIndex !== 'undefined') {
            underscored = prevIndex + '_' + recursionInc;
            obj = {};
            obj[underscored] = jsonObj[i];
            dataArr.push(obj);
            jsonObj[i] = '{{ ' + underscored + ' }}';
          }
        }

        else {
          jsonObj[i] = '{{ ' + underscored + ' }}';
        }
      }
    }
    return jsonObj;
  };

  /**
   * @param {object} jsonForXhtml - JSON for conversion to Mustache syntax.
   * @return {string} XHTML.
   */
  exports.jsonToMustache = function (jsonForXhtml) {
    var xhtml = builder.buildObject(jsonForXhtml);
    // Remove XML declaration.
    xhtml = xhtml.replace(/<\?xml(.|\s)*?\?>/g, '');
    // Replace html tags with Mustache tags.
    xhtml = xhtml.replace('<html>', '{{# html }}').replace('</html>', '{{/ html }}');
    // Clean up.
    xhtml = xhtml.replace(/^\s*\n/g, '');

    return xhtml;
  };

  exports.redirectWithMsg = function (res, type, msg, target, url) {
    if (res) {
      target = typeof target === 'string' ? target : '';
      url = typeof url === 'string' ? url : '';
      res.writeHead(303, {Location: 'html-scraper?' + type + '=' + msg + '&target=' + target + '&url=' + url});
      res.end();
    }
  };

  exports.targetHtmlGet = function ($targetEl, targetIndex, $) {
    // Iterate through the collection of selected elements. If an index
    // is specified, skip until that index is iterated upon.
    var $el;
    var innerHtml;
    var j = 0;
    var outerHtml;
    var targetFirst;
    var targetHtml = '';

    $targetEl.each(function (i, el) {
      if (targetIndex === '' || parseInt(targetIndex, 10) === i) {
        $el = $(el);
        // Cheerio hack for getting outerHTML.
        innerHtml = $el.html();
        outerHtml = $el.html(innerHtml) + '\n';
        targetHtml += outerHtml;
        if (j === 0) {
          targetFirst = outerHtml;
        }
        j++;
      }
    });

    return {all: targetHtml, first: targetFirst};
  };

  exports.targetValidate = function (target, res, req) {
    // Split at array index, if any.
    var targetSplit = target.split('[', 2);
    targetSplit[1] = targetSplit[1] ? targetSplit[1] : '';

    // Validate that targetSplit[0] is a css selector.
    if (!targetSplit[0].match(/^(#|\.)?[a-z][\w#\-\.]*$/i)) {
      exports.redirectWithMsg(res, 'error', 'Incorrect+submission.', req.body.target, req.body.url);
      return [];
    }

    // Remove closing bracket from targetSplit[1] and validate it is an integer.
    targetSplit[1] = targetSplit[1].substr(0, targetSplit[1].length - 1);
    if (!targetSplit[1].match(/\d*/)) {
      exports.redirectWithMsg(res, 'error', 'Incorrect+submission.', req.body.target, req.body.url);
      return [];
    }

    return targetSplit;
  };

  /**
   * Get JSON and array from XHTML.
   *
   * @param {string} targetXhtml - Well formed XHTML.
   * @return {object} Prop 1: json, Prop2: array.
   */
  exports.xhtmlToJsonAndArray = function (targetXhtml) {
    var dataArr;
    var jsonForXhtml;

    // Convert to JSON.
    xml2js.parseString(targetXhtml, function (err, res) {
      if (err) {
        utils.error(err);
        return {};
      }

      // jsonRecurse builds dataArr.
      dataArr = [];
      jsonForXhtml = exports.jsonRecurse(res, dataArr, 0);
    });

    return {json: jsonForXhtml, array: dataArr};
  };

  exports.main = function (req, res) {
    var $;
    var dataArr1;
    var dataObj;
    var dataStr;
    var fileHtml;
    var fileJson;
    var fileName;
    var jsonForData;
    var jsonForXhtml;
    var output;
    var target;
    var targetBase;
    var $targetEl;
    var targetFirst;
    var targetHtml;
    var targetHtmlObj;
    var targetIndex;
    var targetSplit;
    var targetXhtml;
    var templateDir;
    var xhtml = '';

    // HTML scraper action on submission of URL.
    if (typeof req.body.url === 'string' && req.body.url.trim() && typeof req.body.target === 'string') {
      try {
        request(req.body.url, function (error, response, body) {
          if (error || response.statusCode !== 200) {
            exports.redirectWithMsg(res, 'error', 'Not+getting+a+valid+response+from+that+URL.', req.body.target, req.body.url);
            return;
          }

          $ = cheerio.load(body);
          target = req.body.target.trim();
          targetSplit = exports.targetValidate(target, res, req);
          targetBase = targetSplit[0];
          targetIndex = targetSplit[1];
          targetHtml = '';
          $targetEl = $(targetBase);

          if ($targetEl.length) {
            targetHtmlObj = exports.targetHtmlGet($targetEl, targetIndex, $);

            // Sanitize scraped HTML.
            targetHtml = '<html>' + exports.htmlSanitize(targetHtmlObj.all) + '</html>';
            targetFirst = '<html>' + exports.htmlSanitize(targetHtmlObj.first) + '</html>';

            // Convert HTML to XHTML for conversion to full JSON data object.
            targetXhtml = exports.htmlToXhtml(targetHtml);

            // Get array from XHTML.
            dataArr1 = exports.xhtmlToJsonAndArray(targetXhtml).array;

            // Delete html tags.
            targetHtml = targetHtml.replace('<html>', '').replace('</html>', '');

            // Convert HTML to XHTML for Mustache template.
            targetXhtml = exports.htmlToXhtml(targetFirst);

            // Get JSON and array from XHTML.
            dataObj = exports.xhtmlToJsonAndArray(targetXhtml);

            // Get dataArr2 array. We can't use dataArr1 because we need it
            // untouched so we can build jsonForData.
            dataArr1 = dataObj.array;

            // Build XHTML with mustache tags.
            jsonForXhtml = dataObj.json;
            xhtml = exports.jsonToMustache(jsonForXhtml);
          }

          // Convert dataArr1 to JSON and stringify for output.
          jsonForData = exports.dataArrayToJson(dataArr1);
          dataStr = JSON.stringify(jsonForData, null, 2);

          output = '';
          output += htmlObj.head;
          output += '<section>\n';
          output += htmlObj.scraperTitle;
          output += htmlObj.reviewerPrefix;
          // HTML entities.
          output += $('<div/>').text(targetHtml).html().replace(/\n/g, '<br>');
          output += htmlObj.reviewerSuffix;
          output += htmlObj.importerPrefix;
          output += xhtml;
          output += htmlObj.json;
          output += dataStr;
          output += htmlObj.importerSuffix;
          output += htmlObj.landingBody;
          output += '</section>';
          output += htmlObj.foot;
          output = output.replace('{{ title }}', 'Fepper HTML Scraper');
          output = output.replace('{{ class }}', 'scraper');
          output = output.replace('{{ url }}', req.body.url);
          output = output.replace('{{ target }}', req.body.target);
          res.end(output);
        });
      }
      catch (err) {
        utils.error(err);
      }
    }

    // HTML importer action on submission of filename.
    else if (typeof req.body.filename === 'string' && req.body.filename !== '') {
      // Limit filename characters.
      if (!exports.isFilenameValid(req.body.filename)) {
        exports.redirectWithMsg(res, 'error', 'Please+enter+a+valid+filename!.', req.body.target, req.body.url);
        return;
      }
      else {
        fileName = req.body.filename;
      }

      templateDir = 'patternlab-node/source/_patterns/98-scrape';
      fileHtml = exports.newlineFormat(req.body.html);
      fileJson = exports.newlineFormat(req.body.json);
      exports.filesWrite(templateDir, fileName, fileHtml, fileJson, res);
    }

    // If no form variables sent, redirect back with GET.
    else {
      try {
        exports.redirectWithMsg(res, 'error', 'Incorrect+submission.', req.body.target, req.body.url);
        return;
      }
      catch (err) {
        utils.error(err);
      }
    }
  };
})();
Example #27
0
  exports.htmlToXhtml = function (targetHtml) {
    var targetParsed = domParser.parseFromString(targetHtml, 'text/html');
    var targetXhtml = xmlSerializer.serializeToString(targetParsed);

    return targetXhtml;
  };
 var __html = Array.prototype.slice.call(node.childNodes).map(n=> serializer.serializeToString(n)).join('');
    return new Promise((resolve, reject) => {
        // No point in generating when there are no files
        if ( !files.length ) {
            return resolve();
        }

        // Initialize DOM/XML/SVGO
        const DOMParser = new xmldom.DOMParser();
        const XMLSerializer = new xmldom.XMLSerializer();
        const XMLDoc = new xmldom.DOMImplementation().createDocument(null, null, null);
        const SVGOptimizer = new svgo(merge(options.output.svgo, {
            plugins: [
                // Prevent empty var:* attributes from getting removed prematurely
                { removeEmptyAttrs: false },

                // Prevent groups from getting optimized prematurely as they may contain var:* attributes
                { moveGroupAttrsToElems: false },
                { collapseGroups: false },

                // Prevent titles from getting removed prematurely
                { removeTitle: false }
            ]
        }));

        // Create SVG element
        const svg = XMLDoc.createElement('svg');
        const sizes = {
            width: [],
            height: []
        };

        const formatPostfix = (value) => {
            if ( typeof value === 'string' ) {
                return value;
            }

            return '';
        };

        // Add namespaces
        svg.setAttribute('xmlns', 'http://www.w3.org/2000/svg');
        if ( options.sprite.generate.use ) {
            svg.setAttribute('xmlns:xlink', 'http://www.w3.org/1999/xlink');
        }

        Promise.all(files.map((file) => new Promise((resolve) => {
            const prefix = generateSpritePrefix(options.sprite.prefix, file);
            const id = `${prefix}${idify(path.basename(file, path.extname(file)))}`;

            let sprite = fs.readFileSync(file, 'utf8');

            if ( options.output.svgo === false ) {
                return resolve({
                    id: id,
                    sprite: sprite
                });
            }

            if ( hasVariables(sprite) && !hasVarNamespace(sprite) ) {
                sprite = addVarNamespace(sprite);
            }

            SVGOptimizer.optimize(sprite, {
                path: file
            }).then((output) => resolve({
                prefix: prefix,
                file: file,
                id: id,
                sprite: output.data
            }));
        }))).then((items) => {
            // Add the xmlns:var attribute when variables are found in any sprite
            if ( hasVariables(items.map((item) => item.sprite).join('\n')) ) {
                svg.setAttribute(`xmlns:${VAR_NAMESPACE}`, VAR_NAMESPACE_VALUE);
            }

            items.forEach((item) => {
                const sprite = DOMParser.parseFromString(item.sprite).documentElement;

                // Attributes that should be transfered to output SVG
                const attributes = Array.from(sprite.attributes).reduce((attributes, attribute) => {
                    // Blacklist several attributes as they'll be added/removed while parsing
                    if ( ['viewbox', 'width', 'height', 'id', 'xmlns'].includes(attribute.name.toLowerCase()) ) {
                        return attributes;
                    }

                    return [...attributes, {
                        name: attribute.name,
                        value: attribute.value
                    }];
                }, []);

                // Add xmlns:* attributes to root SVG
                attributes.forEach((attribute) => {
                    if ( !attribute.name.toLowerCase().startsWith('xmlns:') ) {
                        return;
                    }

                    svg.setAttribute(attribute.name, attribute.value);
                });

                // Get sizes
                let viewbox = (sprite.getAttribute('viewBox') || sprite.getAttribute('viewbox')).split(' ').map((a) => parseFloat(a));
                let width = parseFloat(sprite.getAttribute('width'));
                let height = parseFloat(sprite.getAttribute('height'));

                if ( viewbox.length !== 4 && ( isNaN(width) || isNaN(height) ) ) {
                    return reject(`Invalid SVG '${item.file}'; it's lacking both a viewBox and width/height attributes...`);
                }

                if ( viewbox.length !== 4 ) {
                    viewbox = [0, 0, width, height];
                }

                if ( isNaN(width) ) {
                    width = viewbox[2];
                }

                if ( isNaN(height) ) {
                    height = viewbox[3];
                }

                // Create symbol
                if ( options.sprite.generate.symbol ) {
                    const symbol = XMLDoc.createElement('symbol');

                    // Attributes
                    symbol.setAttribute('id', `${item.id}${formatPostfix(options.sprite.generate.symbol)}`);
                    symbol.setAttribute('viewBox', viewbox.join(' '));
                    attributes.forEach((attribute) => {
                        if ( !['preserveaspectratio'].includes(attribute.name.toLowerCase()) ) {
                            return;
                        }

                        symbol.setAttribute(attribute.name, attribute.value);
                    });

                    if ( options.sprite.generate.title ) {
                        // Make sure we don't overwrite the existing title
                        const hasTitle = (sprite) => {
                            const titles = Array.from(sprite.childNodes).filter((childNode) => {
                                return childNode.nodeName.toLowerCase() === 'title';
                            });

                            return !!titles.length;
                        };

                        // Add title to improve accessibility
                        if ( !hasTitle(sprite) ) {
                            const title = XMLDoc.createElement('title');
                            title.appendChild(XMLDoc.createTextNode(item.id.replace(item.prefix, '')));
                            symbol.appendChild(title);
                        }
                    }

                    // Clone the original contents of the SVG file into the new symbol
                    Array.from(sprite.childNodes).forEach((childNode) => {
                        symbol.appendChild(childNode);
                    });

                    svg.appendChild(symbol);
                }

                if ( options.sprite.generate.use ) {
                    // Generate <use> elements within spritemap to allow usage within CSS
                    const use = XMLDoc.createElement('use');
                    const y = sizes.height.reduce((a, b) => a + b, 0) + (sizes.height.length * options.sprite.gutter);

                    use.setAttribute('xlink:href', `#${item.id}${formatPostfix(options.sprite.generate.symbol)}`);
                    use.setAttribute('x', '0');
                    use.setAttribute('y', y);
                    use.setAttribute('width', width.toString());
                    use.setAttribute('height', height.toString());
                    svg.appendChild(use);
                }

                if ( options.sprite.generate.view ) {
                    // Generate <view> elements within spritemap to allow usage within CSS
                    const view = XMLDoc.createElement('view');
                    const y = sizes.height.reduce((a, b) => a + b, 0) + (sizes.height.length * options.sprite.gutter);

                    // Attributes
                    view.setAttribute('id', `${item.id}${formatPostfix(options.sprite.generate.view)}`);
                    view.setAttribute('viewBox', `0 ${Math.max(0, y - (options.sprite.gutter / 2))} ${width + (options.sprite.gutter / 2)} ${height + (options.sprite.gutter / 2)}`);
                    attributes.forEach((attribute) => {
                        if ( !['preserveaspectratio'].includes(attribute.name.toLowerCase()) ) {
                            return;
                        }

                        view.setAttribute(attribute.name, attribute.value);
                    });

                    svg.appendChild(view);
                }

                // Update sizes
                sizes.width.push(width);
                sizes.height.push(height);
            });

            if ( options.output.svg.sizes ) {
                // Add width/height to spritemap
                svg.setAttribute('width', Math.max.apply(null, sizes.width).toString());
                svg.setAttribute('height', (sizes.height.reduce((a, b) => a + b, 0) + ((sizes.height.length - 1) * options.sprite.gutter)).toString());
            }

            return resolve(XMLSerializer.serializeToString(svg));
        });
    });
        }))).then((items) => {
            // Add the xmlns:var attribute when variables are found in any sprite
            if ( hasVariables(items.map((item) => item.sprite).join('\n')) ) {
                svg.setAttribute(`xmlns:${VAR_NAMESPACE}`, VAR_NAMESPACE_VALUE);
            }

            items.forEach((item) => {
                const sprite = DOMParser.parseFromString(item.sprite).documentElement;

                // Attributes that should be transfered to output SVG
                const attributes = Array.from(sprite.attributes).reduce((attributes, attribute) => {
                    // Blacklist several attributes as they'll be added/removed while parsing
                    if ( ['viewbox', 'width', 'height', 'id', 'xmlns'].includes(attribute.name.toLowerCase()) ) {
                        return attributes;
                    }

                    return [...attributes, {
                        name: attribute.name,
                        value: attribute.value
                    }];
                }, []);

                // Add xmlns:* attributes to root SVG
                attributes.forEach((attribute) => {
                    if ( !attribute.name.toLowerCase().startsWith('xmlns:') ) {
                        return;
                    }

                    svg.setAttribute(attribute.name, attribute.value);
                });

                // Get sizes
                let viewbox = (sprite.getAttribute('viewBox') || sprite.getAttribute('viewbox')).split(' ').map((a) => parseFloat(a));
                let width = parseFloat(sprite.getAttribute('width'));
                let height = parseFloat(sprite.getAttribute('height'));

                if ( viewbox.length !== 4 && ( isNaN(width) || isNaN(height) ) ) {
                    return reject(`Invalid SVG '${item.file}'; it's lacking both a viewBox and width/height attributes...`);
                }

                if ( viewbox.length !== 4 ) {
                    viewbox = [0, 0, width, height];
                }

                if ( isNaN(width) ) {
                    width = viewbox[2];
                }

                if ( isNaN(height) ) {
                    height = viewbox[3];
                }

                // Create symbol
                if ( options.sprite.generate.symbol ) {
                    const symbol = XMLDoc.createElement('symbol');

                    // Attributes
                    symbol.setAttribute('id', `${item.id}${formatPostfix(options.sprite.generate.symbol)}`);
                    symbol.setAttribute('viewBox', viewbox.join(' '));
                    attributes.forEach((attribute) => {
                        if ( !['preserveaspectratio'].includes(attribute.name.toLowerCase()) ) {
                            return;
                        }

                        symbol.setAttribute(attribute.name, attribute.value);
                    });

                    if ( options.sprite.generate.title ) {
                        // Make sure we don't overwrite the existing title
                        const hasTitle = (sprite) => {
                            const titles = Array.from(sprite.childNodes).filter((childNode) => {
                                return childNode.nodeName.toLowerCase() === 'title';
                            });

                            return !!titles.length;
                        };

                        // Add title to improve accessibility
                        if ( !hasTitle(sprite) ) {
                            const title = XMLDoc.createElement('title');
                            title.appendChild(XMLDoc.createTextNode(item.id.replace(item.prefix, '')));
                            symbol.appendChild(title);
                        }
                    }

                    // Clone the original contents of the SVG file into the new symbol
                    Array.from(sprite.childNodes).forEach((childNode) => {
                        symbol.appendChild(childNode);
                    });

                    svg.appendChild(symbol);
                }

                if ( options.sprite.generate.use ) {
                    // Generate <use> elements within spritemap to allow usage within CSS
                    const use = XMLDoc.createElement('use');
                    const y = sizes.height.reduce((a, b) => a + b, 0) + (sizes.height.length * options.sprite.gutter);

                    use.setAttribute('xlink:href', `#${item.id}${formatPostfix(options.sprite.generate.symbol)}`);
                    use.setAttribute('x', '0');
                    use.setAttribute('y', y);
                    use.setAttribute('width', width.toString());
                    use.setAttribute('height', height.toString());
                    svg.appendChild(use);
                }

                if ( options.sprite.generate.view ) {
                    // Generate <view> elements within spritemap to allow usage within CSS
                    const view = XMLDoc.createElement('view');
                    const y = sizes.height.reduce((a, b) => a + b, 0) + (sizes.height.length * options.sprite.gutter);

                    // Attributes
                    view.setAttribute('id', `${item.id}${formatPostfix(options.sprite.generate.view)}`);
                    view.setAttribute('viewBox', `0 ${Math.max(0, y - (options.sprite.gutter / 2))} ${width + (options.sprite.gutter / 2)} ${height + (options.sprite.gutter / 2)}`);
                    attributes.forEach((attribute) => {
                        if ( !['preserveaspectratio'].includes(attribute.name.toLowerCase()) ) {
                            return;
                        }

                        view.setAttribute(attribute.name, attribute.value);
                    });

                    svg.appendChild(view);
                }

                // Update sizes
                sizes.width.push(width);
                sizes.height.push(height);
            });

            if ( options.output.svg.sizes ) {
                // Add width/height to spritemap
                svg.setAttribute('width', Math.max.apply(null, sizes.width).toString());
                svg.setAttribute('height', (sizes.height.reduce((a, b) => a + b, 0) + ((sizes.height.length - 1) * options.sprite.gutter)).toString());
            }

            return resolve(XMLSerializer.serializeToString(svg));
        });