return new Promise(function (res, rej) {
qpm_queue.promiseWhile(function() {
return goAgain
}, function() {
return new Promise(function(resolve, reject) {
count++
//var uri = target + nextPage
var uri = target
debug(uri)
solidbot.getPage(uri, convertor, interval)
.then(function(ret) {
debug('ret', ret)
// process the data
for (var k in ret) {
if (ret.hasOwnProperty(k)) {
v = ret[k]
debug(k)
debug(v)
if (k === 'urn:string:next') {
continue
}
if (k === 'http://xmlns.com/foaf/0.1/mbox') {
v[0] = 'mailto:' + v[0]
}
debug('end')
turtle += solidbot.toTurtle(k, v, base)
}
}
// paginate
nextPage = getNextPage(ret)
debug('page', page)
debug('nextPage', nextPage)
if (nextPage && !isNaN(parseInt(nextPage)) && nextPage === (page + 1) && count < MAX_PAGES ) {
debug('goAgain')
page = nextPage
goAgain = true
uri = target + nextPage
resolve()
} else {
goAgain = false
resolve()
}
})
})
}).then(function() {
// finally
var unique = solidbot.sortUnique(turtle)
res(unique)
})
})
/**
* version as a command
*/
function bin(argv) {
// setup config
var config = require('../config/config.js')
program
.arguments('<uri> [uris...]')
.option('-d, --database <database>', 'Database')
.parse(argv)
var defaultDatabase = 'media'
config.database = program.database || config.database || defaultDatabase
var uris = program.args;
if (!uris || uris.length === 0) {
return 'You must enter a valid uri'
}
// Usage
var i = 0;
qpm_queue.promiseWhile(function () { return i < uris.length }, function () {
qpm_media.addMedia(uris[i]).then(function(ret) {
console.log(ret)
ret.conn.close()
}).catch(function(err){
console.error(err)
})
console.log(i);
i++;
return Q.delay(1000); // arbitrary async
}).then(function () {
console.log("done");
}).done();
}