-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathgooglescraper.js
93 lines (73 loc) · 2.17 KB
/
googlescraper.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
var links = [];
var authour = [];
var casper1 = require('casper').create();
var casper = require('casper').create();
var last,names;
var list = range(1, 1);
var fs = require('fs');
var utils = require('utils');
function range(start, count) {
if(arguments.length == 1) {
count = start;
start = 0;
}
var foo = [];
for (var i = 0; i < count; i++) {
foo.push(start + i);
}
return foo;
}
function getLinks() {
var links = document.querySelectorAll('a.firstCredit');
return Array.prototype.map.call(links, function(e) {
return e.getAttribute('href');
});
}
casper.start('http://pinterest.com/search/pins/?q=book', function() {
// search for 'casperjs' from google form
// this.fill('form[action="/search/"]', { q: 'book' }, true);
});
casper.on('remote.message', function(msg) {
// authour.push(msg);
});
casper.thenEvaluate(function() {
window.x = 0;
var intervalID = setInterval(function() {
console.log("Using setInternal " + window.x);
if (++window.x === 3) {
window.clearInterval(intervalID);
}
}, 20);
});
casper.each(list, function(self, i) {
self.wait(20, function() {
last = i;
this.page.scrollPosition = { top: this.page.scrollPosition["top"] + 10000, left: 0 };
});
});
casper.waitFor(function() {
return last === list[list.length - 1] && 3 === this.getGlobal('x');
}, function() {
this.echo('All done.')
});
casper.then(function() {
// aggregate results for the 'casperjs' search
// this.fill('form[action="/search/"]', { q: 'pen' }, true);
links = this.evaluate(getLinks);
});
casper.then(function() {
// aggregate results for the 'phantomjs' search
links = links.concat(this.evaluate(getLinks));
});
casper.then(function() {
// aggregate results for the 'phantomjs' search
this.echo(links.length);
casper.each(links.slice(0,4) , function(self, i) {
casper.thenOpen('http://pinterest.com/'+i, function() {
var x = this.getTitle().split(' ');
// names = names.push(x[2]);
});
});
fs.write('outfile.txt','Write this', 'w');
});
casper.run();