let promise = new Promise(function(resolve, reject) {
if (ItemsHelper.ItemNeedToFetchOnAmazon(item)) {
opHelper.execute('ItemSearch', {
'SearchIndex': item.category.amazon_label,
'Keywords': item.label,
'ResponseGroup': 'Images,ItemAttributes'
}, function(err, results) { // you can add a third parameter for the raw xml response, "results" here are currently parsed using xml2js
if (err) {
resolve(item);
} else {
if (results && results.ItemSearchResponse) {
item.amazon.last_fetch = Date.now();
item.amazon.more_results = results.ItemSearchResponse.Items[0].MoreSearchResultsUrl[0];
item.thumbnail = ItemsHelper.GetThumbnailFromAmazon(results.ItemSearchResponse.Items[0].Item);
item.amazon.items = ItemsHelper.FormatAmazonItem(results.ItemSearchResponse.Items[0].Item);
item.save(function(err, item) {
if (err) console.log(err);
resolve(item);
});
} else {
resolve(item);
}
}
});
} else {
resolve(item);
}
});
findItem: function(filter, keywords, callback) {
amazon.execute('ItemSearch', {
'SearchIndex': filter,
'Keywords': keywords,
'ResponseGroup': 'ItemAttributes,Offers,Images'
}).then(callback).catch(callback);
},
module.exports = function(app) {
var opHelper = new OperationHelper({
awsId: 'AKIAJUG5QBQFJYFPLZSQ',
awsSecret: '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',
assocId: 'kpin0c-20'
});
app.get('/products', function(req, resp) {
console.log('KEYWORDS: ', req.query.keywords);
opHelper.execute('ItemSearch', {
'SearchIndex': 'All',
'Keywords': req.query.keywords,
'ResponseGroup': 'ItemAttributes, Images'
}, function(err, results) {
resp.send(_.map(results.ItemSearchResponse.Items[0].Item, function(item) {
var result = {
image: item.MediumImage[0].URL[0],
title: item.ItemAttributes[0].Title[0]
};
_.forEach(item.ItemLinks[0].ItemLink, function(itemLink) {
if (itemLink.Description[0] === 'Add To Wishlist') {
result.wish = itemLink.URL[0];
}
});
return result;
}));
});
});
};
app.get('/products', function(req, resp) {
console.log('KEYWORDS: ', req.query.keywords);
opHelper.execute('ItemSearch', {
'SearchIndex': 'All',
'Keywords': req.query.keywords,
'ResponseGroup': 'ItemAttributes, Images'
}, function(err, results) {
resp.send(_.map(results.ItemSearchResponse.Items[0].Item, function(item) {
var result = {
image: item.MediumImage[0].URL[0],
title: item.ItemAttributes[0].Title[0]
};
_.forEach(item.ItemLinks[0].ItemLink, function(itemLink) {
if (itemLink.Description[0] === 'Add To Wishlist') {
result.wish = itemLink.URL[0];
}
});
return result;
}));
});
});
Operator.prototype.search = function(done) {
var _self = this;
var _conditions = this._conditions();
Operation.execute(this.operationType, _conditions, function(err, res) {
if (err) {
return done(err);
}
if (res.ItemSearchErrorResponse) {
var errorCode = res.ItemSearchErrorResponse.Error[0].Code[0];
switch (errorCode) {
case 'RequestThrottled':
_self.retry++;
setTimeout(function() {
_self.search(done);
}, INTERVAL * _self.retry);
break;
case 'SignatureDoesNotMatch':
done(res.ItemSearchErrorResponse);
break;
}
// エラーコードを記録しておく
if (process.env.NODE_ENV === 'development') {
return log.info(INTERVAL * _self.retry + ':' + errorCode + ':' + _conditions.Author + ':' + _conditions.ItemPage);
}
return log.warn.info(errorCode);
}
_self.retry = 0;
done(null, res.ItemSearchResponse.Items[0]);
});
};
Scraper.prototype.scrapeDetails = function (cb) {
var self = this;
var opHelper = new OperationHelper({
awsId: config.amazon.awsId,
awsSecret: config.amazon.awsSecret,
assocId: config.amazon.assocId,
endPoint: self.amazonEndpoint,
});
opHelper.execute(
"ItemLookup",
{
ItemId: self.isbn,
IdType: "EAN",
SearchIndex: "Books",
ResponseGroup: "ItemAttributes",
},
function (error, apaResults) {
if (error) {
return cb(error, null);
}
// console.log(JSON.stringify(apaResults, null, 2));
var apaResultsItem = apaResults.ItemLookupResponse.Items[0].Item;
// Check that we have a result
if (!apaResultsItem) {
return cb(null, null);
}
var response = apaResultsItem[0].ItemAttributes[0];
// console.log(JSON.stringify(response, null, 2));
var results = {
authors: response.Author || [],
title: response.Title[0] || "",
};
cb(null, results);
}
);
};
}, function(err, results) { // you can add a third parameter for the raw xml response, "results" here are currently parsed using xml2js
if (err) console.error(err);
var itemId = results.ItemSearchResponse.Items[0].Item[0].ASIN;
opHelper.execute('ItemLookup', {
'ItemId': itemId,
'ResponseGroup': 'Images, ItemAttributes'
}, function (err, results) {
res.send(results);
});
});
handler: function (request) {
/* do things with request; */
opHelper.execute('ItemSearch', {
'SearchIndex': 'Books',
'Keywords': 'harry potter',
'ResponseGroup': 'ItemAttributes,Offers'
}, function(results) { // you can add a second parameter here to examine the raw xml response
console.log(results);
});
}
var search = function(option) {
console.log('search:', option.track);
var opHelper = new OperationHelper({
awsId : security.awsId,
awsSecret : security.awsSecret,
assocId : security.assocId,
endPoint : 'ecs.amazonaws.jp'
});
opHelper.execute('ItemSearch', {
'SearchIndex': 'All',
'Keywords': option.track,
'ResponseGroup': 'ItemAttributes,Offers,Images,Reviews,IsEligibleForSuperSaverShipping',
'MerchantId': 'All'
}, function(err, resultXml) {
// http://docs.aws.amazon.com/AWSECommerceService/latest/DG/AnatomyofaResponse.html
parseString(resultXml, function (err, results) {
if(err) {
cosole.log(err);
} else {
if(results.hasOwnProperty('ItemSearchResponse')) {
console.log(results.ItemSearchResponse.Items[0]);
var items = results.ItemSearchResponse.Items[0].Item;
console.log(items);
if(items) {
// results.ItemSearchResponse.Items.forEach(function(item) {;
// console.log(item);
// });
io.sockets.emit('SearchResults', {items: items});
}
} else {
console.log('results doee not have ItemSearchResponse property');
}
}
});
});
};
amazon.delay(function(){
// Have to always create a new opHelper or else the SAX parser barfs.
// Don't know why that is. Should be a TODO.
var opHelper = new OperationHelper({
awsId: aws.AWSAccessKeyId,
awsSecret: aws.AWSSecretKey,
assocId: aws.AWSTagId,
});
opHelper.execute('ItemSearch', {
'SearchIndex': 'All',
'Keywords': data,
}, function(error, results) {
if (error) {
console.log('Error: ' + error + "\n")
callback(error, results.ItemSearchErrorResponse.Error);
}
try {
num = results.ItemSearchResponse.Items[0].TotalResults[0];
} catch (err) {
console.log("Caught Error: " + err + "\nResults: " + results);
callback(err, 'UPCLOOKUPERROR');
}
if (num == 0) {
error = 'No item found matching that UPC.';
ASIN = 'UPCNOTFOUND';
} else if (num >= 1) {
ASIN = results.ItemSearchResponse.Items[0].Item[0].ASIN[0];
}
if (num > 1)
error = 'More than one possible result returned. Using the first ASIN.';
callback(error, ASIN);
});
});
index: function (req, res){
// execute(operation, params, callback)
// operation: select from http://docs.aws.amazon.com/AWSECommerceService/latest/DG/SummaryofA2SOperations.html
// params: parameters for operation (optional)
// callback(err, parsed, raw): callback function handling results. err = potential errors raised from xml2js.parseString() or http.request(). parsed = xml2js parsed response. raw = raw xml response.
opHelper.execute('ItemSearch', {
'SearchIndex': 'Books',
'Keywords': 'game of thrones',
'ResponseGroup': 'Small'
}, function(err, results) { // you can add a third parameter for the raw xml response, "results" here are currently parsed using xml2js
if (err) console.error(err);
var itemId = results.ItemSearchResponse.Items[0].Item[0].ASIN;
opHelper.execute('ItemLookup', {
'ItemId': itemId,
'ResponseGroup': 'Images, ItemAttributes'
}, function (err, results) {
res.send(results);
});
});
}
app.get('/Error', function(req, res){
opHelper.execute ('ItemSearch', {
'SearchIndex': 'Wine',
'Title': 'Wine',
'KeyWords': 'Red Wine, White Wine, Bordeaux',
'MinimumPrice': '9500',
'MaximumPrice': '10000',
'ResponseGroup': 'ItemAttributes, Images, BrowseNodes',
'sort': 'relevance'
}, function(err, results){
if(err) {console.log(err);}
else {
res.json(results);
console.log("I received "+ results.ItemSearchResponse.Items[0].Item.length + " items from Amazon.com!");
}
}
);
});
// Amazonの書籍を調べる for Node.js
// モジュールの取り込み
var OperationHelper = require('apac').OperationHelper;
// 以下、アカウント情報を書き入れる --------- (※1)
var opHelper = new OperationHelper({
awsId: '**********', // アクセスキーID
awsSecret: '**********', // シークレットアクセスキー
assocId: 'text2musiccom-22', // アソシエイトID
endPoint: 'ecs.amazonaws.jp', // 日本のendPointを指定
});
// 書籍を検索する ------------- (※2)
opHelper.execute('ItemSearch', {
'SearchIndex': 'Books',
'BrowseNode': 465610, // 日本の書籍ID
'Keywords': '宮沢賢治',
'ResponseGroup': 'Small,OfferSummary',
'Sort': 'salesrank',
'MinimumPrice': 10,
'MaximumPrice': 8000,
'Sort': 'salesrank'
}, function(err, results, xml) {
if (err) { console.log("error"); return; }
// 結果を表示 --- (※3)
var Items = results.ItemSearchResponse.Items;
console.log(Items);
});
function runQueries (querryArray){
for(var i = 0; i < querryArray.length; i++){
newQuery = {
'SearchIndex': querryArray[i].SearchIndex,
'Title': querryArray[i].Title,
'KeyWords': querryArray[i].KeyWords,
'MinimumPrice': querryArray[i].MinimumPrice,
'MaximumPrice': querryArray[i].MaximumPrice,
'ResponseGroup': querryArray[i].ResponseGroup,
'sort': querryArray[i].sort
}
opHelper.execute ('ItemSearch', newQuery
, function(err, results) {
if(err) {console.log(err);}
else { console.log();console.log(newQuery);console.log();
for (var numOfitems = 0; numOfitems < results.ItemSearchResponse.Items[0].Item.length; numOfitems ++) {
//handle problem with undefined item properties
function undefinedField (field){
var value = String;
if (field == undefined) {value ="";}
else if (field.constructor == Array) {value = field.join(',');}
else if (field.constructor == Object) {value = field;}
return value;
}
//handle problems with undefined prices
function undefinedPrices(price){
if(price == undefined) {return '';}
else {return price[0].Amount[0];}
}
//handle problems for products without a picture
function noPictureHandler(picturelink){
if(picturelink == undefined) {return "";}
else {return picturelink[0].URL[0];}
}
//handle problems for products without manufacturer
function noManufacHandler (manufLink){
if(manufLink == undefined){return "";}
else{return manufLink[0];}
}
//handle problems with undefined title
function noTitle(titleLink){
if(titleLink == undefined){return '';}
else{ return titleLink[0]; }
}
//map http request to mongoose model
var newProduct = new models.products({
id: results.ItemSearchResponse.Items[0].Item[numOfitems].ASIN[0],
//categroy_id = first order browsenodeId from Amazon
//category_id: undefinedField(results.ItemSearchResponse.Items[0].Item[numOfitems].BrowseNodes[0].BrowseNode[0].BrowseNodeId[0]),
title: noTitle (results.ItemSearchResponse.Items[0].Item[numOfitems].ItemAttributes[0].Title),
description: undefinedField (results.ItemSearchResponse.Items[0].Item[numOfitems].ItemAttributes[0].Feature),
image_link: noPictureHandler (results.ItemSearchResponse.Items[0].Item[numOfitems].MediumImage),
//brand = manufacturere field from Amazon
brand: noManufacHandler (results.ItemSearchResponse.Items[0].Item[numOfitems].ItemAttributes[0].Manufacturer),
price: undefinedPrices (results.ItemSearchResponse.Items[0].Item[numOfitems].ItemAttributes[0].ListPrice)
});
newProduct.save(function(err){
if (err) {console.log(err);}
});
console.log(newProduct);
};
}
});
}
console.log("I received "+i*10+" products from Amazon!")
}
test: function(callback) {
amazon.execute('ItemLookup', {
'ItemId': 'B019EU9SWW'
}).then(callback).catch(callback)
}
getCart: function(id, callback) {
amazon.execute('CartGet', {
'CartId': id
}).then(callback).catch(callback)
},
createCart: function(id, callback) {
amazon.execute('CartCreate', {
'Item.1.ASIN': id,
'Item.1.Quantity': 1
}).then(callback).catch(callback);
},
Price.prototype.done = function (cb) {
const that = this
const helper = new apac.OperationHelper({
awsId: this.opts.amazonId,
awsSecret: this.opts.amazonSecret,
assocId: this.opts.associateId,
endPoint: this.opts.endpoint
})
// Make sure we only execute the callback once
cb = _.once(cb)
// Convert op name
const op = this.mode === 'search' ? 'ItemSearch' : 'ItemLookup'
// Populate request object
const req = {
'ResponseGroup': 'Offers,ItemAttributes,Images'
}
let searchIndex = this.opts.searchIndex || 'All'
if (this.opts.minimumPrice) req['MinimumPrice'] = this.opts.minimumPrice
if (this.opts.maximumPrice) req['MaximumPrice'] = this.opts.maximumPrice
if (this.opts.page) req['ItemPage'] = this.opts.page
if (this.opts.browseNode) req['BrowseNode'] = this.opts.browseNode
if (this.mode === 'search') {
_.extend(req, { 'SearchIndex': searchIndex, 'Keywords': this.keywords })
} else if (this.mode === 'lookup') {
if (this.ean) {
_.extend(req, {
'ItemId': this.ean, 'IdType': 'EAN', 'SearchIndex': searchIndex
})
} else {
_.extend(req, { 'ItemId': this.itemId })
}
}
// Run the request
helper.execute(op, req)
.then(res => {
success(res, cb)
})
.catch(err => {
error(err, cb)
})
function success (res, cb) {
const error = extractError(res)
// Process errors
if (error) return cb(new Error(error))
// Find the item root
let root = first(res, '$..Item')
if (root && !Array.isArray(root)) root = [root]
// Extract interesting stuff
const result = root
? root.map(function (x) {
return extract.call(that, x, that.extractions)
})
: []
// format images
if (that.opts.loadImages) {
result.forEach(r => {
// Remove images key if we didn't get any
if (!r.images) {
r.images = []
return
}
let rawImages = r.images
if (!Array.isArray(rawImages)) rawImages = [rawImages]
r.images = rawImages.reduce((images, image) => {
const { TinyImage, LargeImage, HiResImage } = image
if (!TinyImage || !LargeImage || !HiResImage) return images
const img = {
small: TinyImage.URL,
big: LargeImage.URL,
hiRes: HiResImage.URL,
}
images.push(img)
return images
}, [])
})
}
// Apply limits
if (that._limit) {
result = _.first(result, that._limit)
}
if (that._one) {
result = _.first(result)
}
return cb(null, result)
}
function error (err, cb) {
return cb(error)
}
return this
}
Scraper.prototype.scrape = function (cb) {
var self = this;
var entries = this.results.entries = [];
var opHelper = new OperationHelper({
awsId: config.amazon.awsId,
awsSecret: config.amazon.awsSecret,
assocId: config.amazon.assocId,
// http://docs.aws.amazon.com/AWSECommerceService/latest/DG/SOAPEndpoints.html
endPoint: self.amazonEndpoint,
});
opHelper.execute(
"ItemLookup",
{
ItemId: self.isbn,
IdType: "EAN",
SearchIndex: "Books",
Condition: "All",
ResponseGroup: "Offers",
MerchantId: "Amazon", // https://forums.aws.amazon.com/message.jspa?messageID=96815#
},
function (error, apaResults) {
if (error) {
return cb(error, null);
}
// console.log(JSON.stringify(apaResults, null, 2));
var asin = self.extractASIN(apaResults);
var results = self.extractOffers(apaResults);
var base = self.homepage;
var url = asin ? base + "dp/" + asin : base + "s?field-keywords=" + self.isbn;
var basePrice = {
currency: self.currencies[0],
};
_.each(self.shipping.blocks, function (block) {
var offers = {};
_.each(results, function (result) {
var format = _.omit(result, "condition", "isSuperSaver");
var maySuperSaver = result.isSuperSaver && block.superSaverPermitted;
if (maySuperSaver && result.price >= block.superSaverMinimum) {
format.shipping = 0;
format.shippingNote = self.superSaverNote;
} else {
format.shipping = block.amount;
format.shippingNote = block.note;
if (maySuperSaver) {
format.shippingNote += " (Free shipping eligible on orders over " + block.superSaverMinimum + " " + basePrice.currency + ")";
}
}
offers[result.condition] = format;
});
entries.push(_.extend({}, basePrice, {
countries: block.countries,
url: url,
offers: offers,
}));
});
self.cleanup(self.results);
cb(null, self.results);
}
);
};