Repository navigation
Expand file tree
/
Copy pathstats-db.js
More file actions
265 lines (247 loc) · 9.22 KB
/
Copy pathstats-db.js
File metadata and controls
265 lines (247 loc) · 9.22 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
const path = require('path');
const fs = require('fs');
const Database = require('better-sqlite3');
const folders = require('./library/folder-setup');
/**
* Persistent storage for the server's request statistics (issue #255).
*
* The in-memory counters in ServerStats reset every time the server restarts,
* which makes them useless for anything but "what has happened since the last
* deploy". This class writes them to a SQLite database that outlives restarts
* and upgrades.
*
* One row per (module, endpoint, operation) per collection interval, holding
* both numbers:
*
* - the *delta*: what happened in that interval. This is what you graph -
* plot count_delta against time and you have requests per interval without
* any differencing at the query end.
* - the *total*: the all-time cumulative count, carried across restarts by
* reloading it at startup. This is what you quote as "how much has this
* endpoint ever been asked to do", and it means a restart shows up as a
* flat spot in the graph rather than a cliff back to zero.
*
* Rows are only written for keys that saw traffic in the interval, so idle
* endpoints cost nothing. Nothing is pruned: at 10 minute intervals a
* continuously busy (endpoint, operation) pair produces ~52,000 rows a year,
* which SQLite does not notice.
*
* better-sqlite3 (rather than sqlite3) on purpose: the writes are synchronous,
* so the final flush in ServerStats.finishStats() lands on disk before
* server.js calls process.exit(). A handful of inserts every ten minutes is
* well under a millisecond, so this does not put the event loop at risk.
*/
class StatsDatabase {
constructor(config, log) {
this.log = log || console;
this.config = config || {};
this.db = null;
this.path = StatsDatabase.resolvePath(this.config);
}
/**
* Where the database lives: config.database if given (relative paths resolve
* under the data folder's databases directory), otherwise stats.db there.
*/
static resolvePath(config) {
const configured = config && config.database;
if (!configured) {
return path.join(folders.databasesDir(), 'stats.db');
}
return path.isAbsolute(configured) ? configured : path.join(folders.databasesDir(), configured);
}
/**
* Open the database, creating the file and the schema if they aren't there.
*/
open() {
fs.mkdirSync(path.dirname(this.path), {recursive: true});
this.db = new Database(this.path);
this.db.pragma('journal_mode = WAL');
this.db.exec(`
CREATE TABLE IF NOT EXISTS request_counts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
time INTEGER NOT NULL,
interval_ms INTEGER NOT NULL,
module TEXT NOT NULL,
endpoint TEXT NOT NULL,
operation TEXT NOT NULL,
count_delta INTEGER NOT NULL,
count_total INTEGER NOT NULL,
time_delta INTEGER NOT NULL,
time_total INTEGER NOT NULL
)
`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_request_counts_time ON request_counts(time)`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_request_counts_key ON request_counts(module, endpoint, operation, id)`);
// One row per run of the server: when it started serving, and the last
// time it was known to be alive. See ServerStats.overallUptime().
this.db.exec(`
CREATE TABLE IF NOT EXISTS server_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
started INTEGER NOT NULL,
last_seen INTEGER NOT NULL,
version TEXT
)
`);
// version was added after the table first shipped
if (!this.db.prepare(`PRAGMA table_info(server_runs)`).all().some(c => c.name === 'version')) {
this.db.exec(`ALTER TABLE server_runs ADD COLUMN version TEXT`);
}
this.touchRunStatement = this.db.prepare(`UPDATE server_runs SET last_seen = ? WHERE id = ?`);
// Under the launcher (cluster.js): what it saw - processes exiting, being recycled and
// coming back, and what it answered itself. See library/cluster-events.js
this.db.exec(`
CREATE TABLE IF NOT EXISTS cluster_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
time INTEGER NOT NULL,
kind TEXT NOT NULL,
process TEXT,
detail TEXT,
count INTEGER NOT NULL,
ms INTEGER NOT NULL
)
`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_cluster_events_time ON cluster_events(time)`);
this.insertClusterEvent = this.db.prepare(`
INSERT INTO cluster_events (time, kind, process, detail, count, ms) VALUES (?, ?, ?, ?, ?, ?)
`);
this.insertClusterEvents = this.db.transaction((rows) => {
for (const row of rows) {
this.insertClusterEvent.run(row.time, row.kind, row.process, row.detail, row.count, row.ms);
}
});
this.insertStatement = this.db.prepare(`
INSERT INTO request_counts
(time, interval_ms, module, endpoint, operation, count_delta, count_total, time_delta, time_total)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
this.insertAll = this.db.transaction((time, intervalMs, rows) => {
for (const row of rows) {
this.insertStatement.run(time, intervalMs, row.module, row.endpoint, row.operation,
row.countDelta, row.countTotal, row.timeDelta, row.timeTotal);
}
});
return this;
}
/**
* The most recent cumulative total for each key, so counting resumes where
* the last run left off instead of starting again from zero.
*/
loadTotals() {
if (!this.db) {
return [];
}
return this.db.prepare(`
SELECT module, endpoint, operation, count_total AS countTotal, time_total AS timeTotal
FROM request_counts
WHERE id IN (SELECT MAX(id) FROM request_counts GROUP BY module, endpoint, operation)
`).all();
}
/**
* Record that a new run of the server has started serving, and return its
* row id for touchRun().
*/
startRun(now, version = null) {
if (!this.db) {
return null;
}
return this.db.prepare(`INSERT INTO server_runs (started, last_seen, version) VALUES (?, ?, ?)`).run(now, now, version).lastInsertRowid;
}
/**
* The runs that were alive at any point since `since`, oldest first, plus
* the last run that ended before then (so a caller can tell whether the
* first run in the period was an upgrade). Each is {id, started, lastSeen,
* version}.
*/
loadRuns(since) {
if (!this.db) {
return [];
}
const before = this.db.prepare(`
SELECT id, started, last_seen AS lastSeen, version FROM server_runs
WHERE last_seen < ? ORDER BY last_seen DESC LIMIT 1
`).get(since);
const during = this.db.prepare(`
SELECT id, started, last_seen AS lastSeen, version FROM server_runs
WHERE last_seen >= ? ORDER BY started
`).all(since);
return before ? [before, ...during] : during;
}
/**
* The server is still alive: move the run's last_seen forward.
*/
touchRun(runId, now) {
if (this.db && runId != null) {
this.touchRunStatement.run(now, runId);
}
}
/**
* How long the server has been running over all the runs recorded before
* this one: the earliest start, and the total time between each run's start
* and the last time it was seen alive. {firstStart: null, runningMs: 0} if
* there are no earlier runs.
*/
loadRunHistory(excludeRunId) {
if (!this.db) {
return {firstStart: null, runningMs: 0};
}
const row = this.db.prepare(`
SELECT MIN(started) AS firstStart, COALESCE(SUM(last_seen - started), 0) AS runningMs
FROM server_runs
WHERE id IS NOT ?
`).get(excludeRunId == null ? -1 : excludeRunId);
return {firstStart: row.firstStart, runningMs: row.runningMs};
}
/**
* The requests counted against a module in (since, now], by endpoint and
* operation: [{endpoint, operation, count}]. A row is attributed to the
* time it was written, so the edges of the period are as fine as the
* collection interval (and ServerStats.flushNow() sharpens the near edge).
*/
countsSince(since, now, module) {
if (!this.db) {
return [];
}
return this.db.prepare(`
SELECT endpoint, operation, SUM(count_delta) AS count
FROM request_counts
WHERE module = ? AND time > ? AND time <= ?
GROUP BY endpoint, operation
`).all(module, since, now);
}
/**
* Record what the launcher saw (rows as library/cluster-events.js validRow gives them).
*/
writeClusterEvents(rows) {
if (this.db && rows && rows.length > 0) {
this.insertClusterEvents(rows);
}
}
/**
* What the launcher saw in (since, now], oldest first: [{time, kind, process, detail, count, ms}].
*/
clusterEventsSince(since, now) {
if (!this.db) {
return [];
}
return this.db.prepare(`
SELECT time, kind, process, detail, count, ms FROM cluster_events
WHERE time > ? AND time <= ? ORDER BY time, id
`).all(since, now);
}
/**
* Write one interval's worth of rows. All or nothing - if the transaction
* fails the caller keeps its deltas and tries again next interval.
*/
write(time, intervalMs, rows) {
if (this.db && rows && rows.length > 0) {
this.insertAll(time, intervalMs, rows);
}
}
close() {
if (this.db) {
this.db.close();
this.db = null;
}
}
}
module.exports = StatsDatabase;