1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
// Important Add your access token here default rate of github is limited to 60 API calls per hour
var access_token = '*';
// Important set the delta threshold for repo details updation. For instance if the threshold is
// set to 1 day(60 * 60 * 24), the cronjob will only update the row if the difference between current
// time and last_updated time stamp of a repo is greater than one day
var delta = 60 * 60 * 24;
var github = require('octonode');
db_pool = require('./database').pool;
async = require('async');
var current_time = Math.floor(new Date().getTime() / 1000);//toISOString().slice(0, 19).replace('T', ' ');
console.log(current_time);
var get_val_from_header = function(header_link) {
// small hack by parsing the header and setting per_page = 1, hence no pagination fetch required
result_intermediate = header_link.split(';');
result_intermediate = result_intermediate[result_intermediate.length - 2];
var reg = /&page=([0-9].*)>/g;
var match = reg.exec(result_intermediate);
return parseInt(match[1]);
}
var get_stargazers = function(result, ghrepo, primary_callback, cb) {
ghrepo.stargazers({per_page: 1}, function(err, data, headers) {
//console.log(JSON.stringify(data));
try {
result['no_of_stars'] = get_val_from_header(headers['link']);
cb(null, result, ghrepo, primary_callback);
} catch(err) {
result['no_of_stars'] = null;
cb(null, result, ghrepo, primary_callback);
}
});
}
var get_branches = function(result, ghrepo, primary_callback, cb) {
ghrepo.branches({per_page: 1}, function(err, data, headers) {
try {
result['versions'] = get_val_from_header(headers['link']);
cb(null, result, ghrepo, primary_callback);
} catch(err) {
result['versions'] = null;
cb(null, result, ghrepo, primary_callback);
}
});
}
var get_contributors = function(result, ghrepo, primary_callback, cb) {
ghrepo.contributors({per_page: 1}, function(err, data, headers) {
try {
result['no_of_developers'] = get_val_from_header(headers['link']);
cb(null, result, primary_callback);
} catch(err) {
result['no_of_developers'] = null;
cb(null, result, primary_callback);
}
});
}
var get_lines_of_code = function(result, cb) {
// #TODO
}
var secondary_callback = function (err, result, primary_callback) {
console.log(result);
if((result['last_updated'] == null) || (current_time - result['last_updated'] > delta)) {
db_pool.getConnection(function(err, connection) {
//Use the connection
var last_updated = current_time;
var no_of_stars = result['no_of_stars'];
var versions = result['versions'];
var no_of_developers = result['no_of_developers'];
sql_query = 'update vnf set last_updated = FROM_UNIXTIME(' + last_updated;
sql_query += '), no_of_stars = ' + no_of_stars + ', versions = ' + versions;
sql_query += ', no_of_developers = ' + no_of_developers + ' where vnf_id = ';
sql_query += result['vnf_id'];
console.log(sql_query);
connection.query(sql_query, function (error, results, fields) {
if (error) throw error;
//And done with the connection.
primary_callback(null, result['vnf_id'] + ' updated');
connection.release();
// Handle error after the release.
// Don't use the connection here, it has been returned to the pool.
});
});
} else {
primary_callback(null, result['vnf_id'] + ' not updated');
}
}
var get_stats = function(vnf_details, callback) {
repo = vnf_details['repo_url'];
repo = repo.split("/");
github_id = repo[repo.length - 2] + '/' + repo[repo.length - 1];
var async = require('async');
var client = github.client(access_token);
var ghrepo = client.repo(github_id);
result = {}
result['vnf_id'] = vnf_details['vnf_id'];
result['last_updated'] = vnf_details['last_updated'];
async.waterfall([
async.apply(get_stargazers, result, ghrepo, callback),
get_branches,
get_contributors,
//get_lines_of_code,
], secondary_callback);
}
db_pool.getConnection(function(err, connection) {
sql_query = 'select vnf_id, repo_url, UNIX_TIMESTAMP(last_updated) last_updated from vnf';
console.log(sql_query);
connection.query(sql_query, function (error, results, fields) {
if (error) throw error;
async.map(results, get_stats, function(error, results) {
//console.log(results);
console.log(results);
process.exit();
});
});
});
|
