function Datastore(options) {
var filename;
// Retrocompatibility with v0.6 and before
if (typeof options === 'string') {
filename = options;
this.inMemoryOnly = false; // Default
} else {
options = options || {};
filename = options.filename;
this.inMemoryOnly = options.inMemoryOnly || false;
this.autoload = options.autoload || false;
this.timestampData = options.timestampData || false;
}
// Determine whether in memory or persistent
if (!filename || typeof filename !== 'string' || filename.length === 0) {
this.filename = null;
this.inMemoryOnly = true;
} else {
this.filename = filename;
}
// String comparison function
this.compareStrings = options.compareStrings;
// Persistence handling
this.persistence = new Persistence({ db: this, nodeWebkitAppName: options.nodeWebkitAppName
, afterSerialization: options.afterSerialization
, beforeDeserialization: options.beforeDeserialization
, corruptAlertThreshold: options.corruptAlertThreshold
});
// This new executor is ready if we don't use persistence
// If we do, it will only be ready once loadDatabase is called
this.executor = new Executor();
if (this.inMemoryOnly) { this.executor.ready = true; }
// Indexed by field name, dot notation can be used
// _id is always indexed and since _ids are generated randomly the underlying
// binary is always well-balanced
this.indexes = {};
this.indexes._id = new Index({ fieldName: '_id', unique: true });
this.ttlIndexes = {};
// Queue a load of the database right away and call the onload handler
// By default (no onload handler), if there is an error there, no operation will be possible so warn the user by throwing an exception
if (this.autoload) { this.loadDatabase(options.onload || function (err) {
if (err) { throw err; }
}); }
}n/a
function Cursor(db, query, execFn) {
this.db = db;
this.query = query || {};
if (execFn) { this.execFn = execFn; }
}n/a
function Datastore(options) {
var filename;
// Retrocompatibility with v0.6 and before
if (typeof options === 'string') {
filename = options;
this.inMemoryOnly = false; // Default
} else {
options = options || {};
filename = options.filename;
this.inMemoryOnly = options.inMemoryOnly || false;
this.autoload = options.autoload || false;
this.timestampData = options.timestampData || false;
}
// Determine whether in memory or persistent
if (!filename || typeof filename !== 'string' || filename.length === 0) {
this.filename = null;
this.inMemoryOnly = true;
} else {
this.filename = filename;
}
// String comparison function
this.compareStrings = options.compareStrings;
// Persistence handling
this.persistence = new Persistence({ db: this, nodeWebkitAppName: options.nodeWebkitAppName
, afterSerialization: options.afterSerialization
, beforeDeserialization: options.beforeDeserialization
, corruptAlertThreshold: options.corruptAlertThreshold
});
// This new executor is ready if we don't use persistence
// If we do, it will only be ready once loadDatabase is called
this.executor = new Executor();
if (this.inMemoryOnly) { this.executor.ready = true; }
// Indexed by field name, dot notation can be used
// _id is always indexed and since _ids are generated randomly the underlying
// binary is always well-balanced
this.indexes = {};
this.indexes._id = new Index({ fieldName: '_id', unique: true });
this.ttlIndexes = {};
// Queue a load of the database right away and call the onload handler
// By default (no onload handler), if there is an error there, no operation will be possible so warn the user by throwing an exception
if (this.autoload) { this.loadDatabase(options.onload || function (err) {
if (err) { throw err; }
}); }
}n/a
function Executor() {
this.buffer = [];
this.ready = false;
// This queue will execute all commands, one-by-one in order
this.queue = async.queue(function (task, cb) {
var newArguments = [];
// task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array
for (var i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]); }
var lastArg = task.arguments[task.arguments.length - 1];
// Always tell the queue task is complete. Execute callback if any was given.
if (typeof lastArg === 'function') {
// Callback was supplied
newArguments[newArguments.length - 1] = function () {
if (typeof setImmediate === 'function') {
setImmediate(cb);
} else {
process.nextTick(cb);
}
lastArg.apply(null, arguments);
};
} else if (!lastArg && task.arguments.length !== 0) {
// false/undefined/null supplied as callbback
newArguments[newArguments.length - 1] = function () { cb(); };
} else {
// Nothing supplied as callback
newArguments.push(function () { cb(); });
}
task.fn.apply(task.this, newArguments);
}, 1);
}n/a
function Persistence(options) {
var i, j, randomString;
this.db = options.db;
this.inMemoryOnly = this.db.inMemoryOnly;
this.filename = this.db.filename;
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw new Error("The datafile name can't end with a ~, which is reserved for crash safe backup files");
}
// After serialization and before deserialization hooks with some basic sanity checks
if (options.afterSerialization && !options.beforeDeserialization) {
throw new Error("Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent
dataloss");
}
if (!options.afterSerialization && options.beforeDeserialization) {
throw new Error("Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent
dataloss");
}
this.afterSerialization = options.afterSerialization || function (s) { return s; };
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s; };
for (i = 1; i < 30; i += 1) {
for (j = 0; j < 10; j += 1) {
randomString = customUtils.uid(i);
if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
throw new Error("beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent
dataloss");
}
}
}
// For NW apps, store data in the same directory where NW stores application data
if (this.filename && options.nodeWebkitAppName) {
console.log("==================================================================");
console.log("WARNING: The nodeWebkitAppName option is deprecated");
console.log("To get the path to the directory where Node Webkit stores the data");
console.log("for your app, use the internal nw.gui module like this");
console.log("require('nw.gui').App.dataPath");
console.log("See https://github.com/rogerwang/node-webkit/issues/500");
console.log("==================================================================");
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
}
}n/a
function EventEmitter() {
EventEmitter.init.call(this);
}n/a
findDocs = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
profiler.step("Finding " + n + " documents");
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT (find) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished finding ' + n + ' docs');
return cb();
}
d.find({ docNumber: order[i] }, function (err, docs) {
if (docs.length !== 1 || docs[0].docNumber !== order[i]) { return cb('One find didnt work'); }
executeAsap(function () {
runFrom(i + 1);
});
});
}
runFrom(0);
}n/a
findDocsWithIn = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
, ins = [], i, j
, arraySize = Math.min(10, n) // The array for $in needs to be smaller than n (inclusive)
;
// Preparing all the $in arrays, will take some time
for (i = 0; i < n; i += 1) {
ins[i] = [];
for (j = 0; j < arraySize; j += 1) {
ins[i].push((i + j) % n);
}
}
profiler.step("Finding " + n + " documents WITH $IN OPERATOR");
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT (find with in selector) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/
s");
profiler.step('Finished finding ' + n + ' docs');
return cb();
}
d.find({ docNumber: { $in: ins[i] } }, function (err, docs) {
if (docs.length !== arraySize) { return cb('One find didnt work'); }
executeAsap(function () {
runFrom(i + 1);
});
});
}
runFrom(0);
}n/a
findOneDocs = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
profiler.step("FindingOne " + n + " documents");
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT (findOne) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished finding ' + n + ' docs');
return cb();
}
d.findOne({ docNumber: order[i] }, function (err, doc) {
if (!doc || doc.docNumber !== order[i]) { return cb('One find didnt work'); }
executeAsap(function () {
runFrom(i + 1);
});
});
}
runFrom(0);
}n/a
getConfiguration = function (benchDb) {
var d, n
, program = require('commander')
;
program
.option('-n --number [number]', 'Size of the collection to test on', parseInt)
.option('-i --with-index', 'Use an index')
.option('-m --in-memory', 'Test with an in-memory only store')
.parse(process.argv);
n = program.number || 10000;
console.log("----------------------------");
console.log("Test with " + n + " documents");
console.log(program.withIndex ? "Use an index" : "Don't use an index");
console.log(program.inMemory ? "Use an in-memory datastore" : "Use a persistent datastore");
console.log("----------------------------");
d = new Datastore({ filename: benchDb
, inMemoryOnly: program.inMemory
});
return { n: n, d: d, program: program };
}...
, benchDb = 'workspace/find.bench.db'
, fs = require('fs')
, path = require('path')
, async = require('async')
, execTime = require('exec-time')
, profiler = new execTime('FIND BENCH')
, commonUtilities = require('./commonUtilities')
, config = commonUtilities.getConfiguration(benchDb)
, d = config.d
, n = config.n
;
async.waterfall([
async.apply(commonUtilities.prepareDb, benchDb)
, function (cb) {
...function getRandomArray(n) {
var res = []
, i, j, temp
;
for (i = 0; i < n; i += 1) { res[i] = i; }
for (i = n - 1; i >= 1; i -= 1) {
j = Math.floor((i + 1) * Math.random());
temp = res[i];
res[i] = res[j];
res[j] = temp;
}
return res;
}n/a
insertDocs = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
profiler.step('Begin inserting ' + n + ' docs');
function runFrom(i) {
if (i === n) { // Finished
var opsPerSecond = Math.floor(1000* n / profiler.elapsedSinceLastStep());
console.log("===== RESULT (insert) ===== " + opsPerSecond + " ops/s");
profiler.step('Finished inserting ' + n + ' docs');
profiler.insertOpsPerSecond = opsPerSecond;
return cb();
}
d.insert({ docNumber: order[i] }, function (err) {
executeAsap(function () {
runFrom(i + 1);
});
});
}
runFrom(0);
}n/a
loadDatabase = function (d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
profiler.step("Loading the database " + n + " times");
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished loading a database' + n + ' times');
return cb();
}
d.loadDatabase(function (err) {
executeAsap(function () {
runFrom(i + 1);
});
});
}
runFrom(0);
}...
var Datastore = require('nedb')
, db = new Datastore();
// Type 2: Persistent datastore with manual loading
var Datastore = require('nedb')
, db = new Datastore({ filename: 'path/to/datafile' });
db.loadDatabase(function (err) { // Callback is optional
// Now commands will be executed
});
// Type 3: Persistent datastore with automatic loading
var Datastore = require('nedb')
, db = new Datastore({ filename: 'path/to/datafile', autoload: true });
...prepareDb = function (filename, cb) {
Persistence.ensureDirectoryExists(path.dirname(filename), function () {
fs.exists(filename, function (exists) {
if (exists) {
fs.unlink(filename, cb);
} else { return cb(); }
});
});
}n/a
removeDocs = function (options, d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
profiler.step("Removing " + n + " documents");
function runFrom(i) {
if (i === n) { // Finished
// opsPerSecond corresponds to 1 insert + 1 remove, needed to keep collection size at 10,000
// We need to subtract the time taken by one insert to get the time actually taken by one remove
var opsPerSecond = Math.floor(1000 * n / profiler.elapsedSinceLastStep());
var removeOpsPerSecond = Math.floor(1 / ((1 / opsPerSecond) - (1 / profiler.insertOpsPerSecond)))
console.log("===== RESULT (remove) ===== " + removeOpsPerSecond + " ops/s");
profiler.step('Finished removing ' + n + ' docs');
return cb();
}
d.remove({ docNumber: order[i] }, options, function (err, nr) {
if (err) { return cb(err); }
if (nr !== 1) { return cb('One remove didnt work'); }
d.insert({ docNumber: order[i] }, function (err) { // We need to reinsert the doc so that we keep the collection's size
at n
// So actually we're calculating the average time taken by one insert
+ one remove
executeAsap(function () {
runFrom(i + 1);
});
});
});
}
runFrom(0);
}n/a
updateDocs = function (options, d, n, profiler, cb) {
var beg = new Date()
, order = getRandomArray(n)
;
profiler.step("Updating " + n + " documents");
function runFrom(i) {
if (i === n) { // Finished
console.log("===== RESULT (update) ===== " + Math.floor(1000* n / profiler.elapsedSinceLastStep()) + " ops/s");
profiler.step('Finished updating ' + n + ' docs');
return cb();
}
// Will not actually modify the document but will take the same time
d.update({ docNumber: order[i] }, { docNumber: order[i] }, options, function (err, nr) {
if (err) { return cb(err); }
if (nr !== 1) { return cb('One update didnt work'); }
executeAsap(function () {
runFrom(i + 1);
});
});
}
runFrom(0);
}n/a
function Cursor(db, query, execFn) {
this.db = db;
this.query = query || {};
if (execFn) { this.execFn = execFn; }
}n/a
_exec = function (_callback) {
var res = [], added = 0, skipped = 0, self = this
, error = null
, i, keys, key
;
function callback (error, res) {
if (self.execFn) {
return self.execFn(error, res, _callback);
} else {
return _callback(error, res);
}
}
this.db.getCandidates(this.query, function (err, candidates) {
if (err) { return callback(err); }
try {
for (i = 0; i < candidates.length; i += 1) {
if (model.match(candidates[i], self.query)) {
// If a sort is defined, wait for the results to be sorted before applying limit and skip
if (!self._sort) {
if (self._skip && self._skip > skipped) {
skipped += 1;
} else {
res.push(candidates[i]);
added += 1;
if (self._limit && self._limit <= added) { break; }
}
} else {
res.push(candidates[i]);
}
}
}
} catch (err) {
return callback(err);
}
// Apply all sorts
if (self._sort) {
keys = Object.keys(self._sort);
// Sorting
var criteria = [];
for (i = 0; i < keys.length; i++) {
key = keys[i];
criteria.push({ key: key, direction: self._sort[key] });
}
res.sort(function(a, b) {
var criterion, compare, i;
for (i = 0; i < criteria.length; i++) {
criterion = criteria[i];
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion.key), model.getDotValue(b, criterion
.key), self.db.compareStrings);
if (compare !== 0) {
return compare;
}
}
return 0;
});
// Applying limit and skip
var limit = self._limit || res.length
, skip = self._skip || 0;
res = res.slice(skip, skip + limit);
}
// Apply projection
try {
res = self.project(res);
} catch (e) {
error = e;
res = undefined;
}
return callback(error, res);
});
}...
async.waterfall([
function (cb) { // If upsert option is set, check whether we need to insert the doc
if (!upsert) { return cb(); }
// Need to use an internal function not tied to the executor to avoid deadlock
var cursor = new Cursor(self, query);
cursor.limit(1)._exec(function (err, docs) {
if (err) { return callback(err); }
if (docs.length === 1) {
return cb();
} else {
var toBeInserted;
try {
...exec = function () {
this.db.executor.push({ this: this, fn: this._exec, arguments: arguments });
}...
// Let's say the database contains these 4 documents
// doc1 = { _id: 'id1', planet: 'Mars', system: 'solar', inhabited: false, satellites: ['Phobos
', 'Deimos'] }
// doc2 = { _id: 'id2', planet: 'Earth', system: 'solar', inhabited: true, humans: { genders: 2, eyes
: true } }
// doc3 = { _id: 'id3', planet: 'Jupiter', system: 'solar', inhabited: false }
// doc4 = { _id: 'id4', planet: 'Omicron Persei 8', system: 'futurama', inhabited: true, humans: {
genders: 7 } }
// No query used means all results are returned (before the Cursor modifiers)
db.find({}).sort({ planet: 1 }).skip(1).limit(2).exec(function (err, docs) {
// docs is [doc3, doc1]
});
// You can sort in reverse order like this
db.find({ system: 'solar' }).sort({ planet: -1 }).exec(function (err, docs) {
// docs is [doc1, doc3, doc2]
});
...limit = function (limit) {
this._limit = limit;
return this;
}...
// Let's say the database contains these 4 documents
// doc1 = { _id: 'id1', planet: 'Mars', system: 'solar', inhabited: false, satellites: ['Phobos
', 'Deimos'] }
// doc2 = { _id: 'id2', planet: 'Earth', system: 'solar', inhabited: true, humans: { genders: 2, eyes
: true } }
// doc3 = { _id: 'id3', planet: 'Jupiter', system: 'solar', inhabited: false }
// doc4 = { _id: 'id4', planet: 'Omicron Persei 8', system: 'futurama', inhabited: true, humans: {
genders: 7 } }
// No query used means all results are returned (before the Cursor modifiers)
db.find({}).sort({ planet: 1 }).skip(1).limit(2).exec(function (err, docs) {
// docs is [doc3, doc1]
});
// You can sort in reverse order like this
db.find({ system: 'solar' }).sort({ planet: -1 }).exec(function (err, docs) {
// docs is [doc1, doc3, doc2]
});
...project = function (candidates) {
var res = [], self = this
, keepId, action, keys
;
if (this._projection === undefined || Object.keys(this._projection).length === 0) {
return candidates;
}
keepId = this._projection._id === 0 ? false : true;
this._projection = _.omit(this._projection, '_id');
// Check for consistency
keys = Object.keys(this._projection);
keys.forEach(function (k) {
if (action !== undefined && self._projection[k] !== action) { throw new Error("Can't both keep and omit fields except for _id
"); }
action = self._projection[k];
});
// Do the actual projection
candidates.forEach(function (candidate) {
var toPush;
if (action === 1) { // pick-type projection
toPush = { $set: {} };
keys.forEach(function (k) {
toPush.$set[k] = model.getDotValue(candidate, k);
if (toPush.$set[k] === undefined) { delete toPush.$set[k]; }
});
toPush = model.modify({}, toPush);
} else { // omit-type projection
toPush = { $unset: {} };
keys.forEach(function (k) { toPush.$unset[k] = true });
toPush = model.modify(candidate, toPush);
}
if (keepId) {
toPush._id = candidate._id;
} else {
delete toPush._id;
}
res.push(toPush);
});
return res;
}...
, skip = self._skip || 0;
res = res.slice(skip, skip + limit);
}
// Apply projection
try {
res = self.project(res);
} catch (e) {
error = e;
res = undefined;
}
return callback(error, res);
});
...projection = function (projection) {
this._projection = projection;
return this;
}...
// Failure: using both modes at the same time
db.find({ planet: 'Mars' }, { planet: 0, system: 1 }, function (err, docs) {
// err is the error message, docs is undefined
});
// You can also use it in a Cursor way but this syntax is not compatible with MongoDB
db.find({ planet: 'Mars' }).projection({ planet: 1, system: 1 }).exec(function
(err, docs) {
// docs is [{ planet: 'Mars', system: 'solar', _id: 'id1' }]
});
// Project on a nested document
db.findOne({ planet: 'Earth' }).projection({ planet: 1, 'humans.genders': 1 }).exec(function (err, doc) {
// doc is { planet: 'Earth', _id: 'id2', humans: { genders: 2 } }
});
...skip = function (skip) {
this._skip = skip;
return this;
}...
// Let's say the database contains these 4 documents
// doc1 = { _id: 'id1', planet: 'Mars', system: 'solar', inhabited: false, satellites: ['Phobos
', 'Deimos'] }
// doc2 = { _id: 'id2', planet: 'Earth', system: 'solar', inhabited: true, humans: { genders: 2, eyes
: true } }
// doc3 = { _id: 'id3', planet: 'Jupiter', system: 'solar', inhabited: false }
// doc4 = { _id: 'id4', planet: 'Omicron Persei 8', system: 'futurama', inhabited: true, humans: {
genders: 7 } }
// No query used means all results are returned (before the Cursor modifiers)
db.find({}).sort({ planet: 1 }).skip(1).limit(2).exec(function (err, docs) {
// docs is [doc3, doc1]
});
// You can sort in reverse order like this
db.find({ system: 'solar' }).sort({ planet: -1 }).exec(function (err, docs) {
// docs is [doc1, doc3, doc2]
});
...sort = function (sortQuery) {
this._sort = sortQuery;
return this;
}...
// Let's say the database contains these 4 documents
// doc1 = { _id: 'id1', planet: 'Mars', system: 'solar', inhabited: false, satellites: ['Phobos
', 'Deimos'] }
// doc2 = { _id: 'id2', planet: 'Earth', system: 'solar', inhabited: true, humans: { genders: 2, eyes
: true } }
// doc3 = { _id: 'id3', planet: 'Jupiter', system: 'solar', inhabited: false }
// doc4 = { _id: 'id4', planet: 'Omicron Persei 8', system: 'futurama', inhabited: true, humans: {
genders: 7 } }
// No query used means all results are returned (before the Cursor modifiers)
db.find({}).sort({ planet: 1 }).skip(1).limit(2).exec(function (err, docs) {
// docs is [doc3, doc1]
});
// You can sort in reverse order like this
db.find({ system: 'solar' }).sort({ planet: -1 }).exec(function (err, docs) {
// docs is [doc1, doc3, doc2]
});
...function uid(len) {
return crypto.randomBytes(Math.ceil(Math.max(8, len * 2)))
.toString('base64')
.replace(/[+\/]/g, '')
.slice(0, len);
}...
});
};
/**
* Create a new _id that's not already in use
*/
Datastore.prototype.createNewId = function () {
var tentativeId = customUtils.uid(16);
// Try as many times as needed to get an unused _id. As explained in customUtils, the probability of this ever happening is extremely
small, so this is O(1)
if (this.indexes._id.getMatching(tentativeId).length > 0) {
tentativeId = this.createNewId();
}
return tentativeId;
};
...function Datastore(options) {
var filename;
// Retrocompatibility with v0.6 and before
if (typeof options === 'string') {
filename = options;
this.inMemoryOnly = false; // Default
} else {
options = options || {};
filename = options.filename;
this.inMemoryOnly = options.inMemoryOnly || false;
this.autoload = options.autoload || false;
this.timestampData = options.timestampData || false;
}
// Determine whether in memory or persistent
if (!filename || typeof filename !== 'string' || filename.length === 0) {
this.filename = null;
this.inMemoryOnly = true;
} else {
this.filename = filename;
}
// String comparison function
this.compareStrings = options.compareStrings;
// Persistence handling
this.persistence = new Persistence({ db: this, nodeWebkitAppName: options.nodeWebkitAppName
, afterSerialization: options.afterSerialization
, beforeDeserialization: options.beforeDeserialization
, corruptAlertThreshold: options.corruptAlertThreshold
});
// This new executor is ready if we don't use persistence
// If we do, it will only be ready once loadDatabase is called
this.executor = new Executor();
if (this.inMemoryOnly) { this.executor.ready = true; }
// Indexed by field name, dot notation can be used
// _id is always indexed and since _ids are generated randomly the underlying
// binary is always well-balanced
this.indexes = {};
this.indexes._id = new Index({ fieldName: '_id', unique: true });
this.ttlIndexes = {};
// Queue a load of the database right away and call the onload handler
// By default (no onload handler), if there is an error there, no operation will be possible so warn the user by throwing an exception
if (this.autoload) { this.loadDatabase(options.onload || function (err) {
if (err) { throw err; }
}); }
}n/a
function EventEmitter() {
EventEmitter.init.call(this);
}n/a
_insert = function (newDoc, cb) {
var callback = cb || function () {}
, preparedDoc
;
try {
preparedDoc = this.prepareDocumentForInsertion(newDoc)
this._insertInCache(preparedDoc);
} catch (e) {
return callback(e);
}
this.persistence.persistNewState(util.isArray(preparedDoc) ? preparedDoc : [preparedDoc], function (err) {
if (err) { return callback(err); }
return callback(null, model.deepCopy(preparedDoc));
});
}...
try {
toBeInserted = model.modify(model.deepCopy(query, true), updateQuery);
} catch (err) {
return callback(err);
}
}
return self._insert(toBeInserted, function (err, newDoc) {
if (err) { return callback(err); }
return callback(null, 1, newDoc, true);
});
}
});
}
, function () { // Perform the update
..._insertInCache = function (preparedDoc) {
if (util.isArray(preparedDoc)) {
this._insertMultipleDocsInCache(preparedDoc);
} else {
this.addToIndexes(preparedDoc);
}
}...
Datastore.prototype._insert = function (newDoc, cb) {
var callback = cb || function () {}
, preparedDoc
;
try {
preparedDoc = this.prepareDocumentForInsertion(newDoc)
this._insertInCache(preparedDoc);
} catch (e) {
return callback(e);
}
this.persistence.persistNewState(util.isArray(preparedDoc) ? preparedDoc : [preparedDoc], function (err) {
if (err) { return callback(err); }
return callback(null, model.deepCopy(preparedDoc));
..._insertMultipleDocsInCache = function (preparedDocs) {
var i, failingI, error;
for (i = 0; i < preparedDocs.length; i += 1) {
try {
this.addToIndexes(preparedDocs[i]);
} catch (e) {
error = e;
failingI = i;
break;
}
}
if (error) {
for (i = 0; i < failingI; i += 1) {
this.removeFromIndexes(preparedDocs[i]);
}
throw error;
}
}...
/**
* If newDoc is an array of documents, this will insert all documents in the cache
* @api private
*/
Datastore.prototype._insertInCache = function (preparedDoc) {
if (util.isArray(preparedDoc)) {
this._insertMultipleDocsInCache(preparedDoc);
} else {
this.addToIndexes(preparedDoc);
}
};
/**
* If one insertion fails (e.g. because of a unique constraint), roll back all previous
..._remove = function (query, options, cb) {
var callback
, self = this, numRemoved = 0, removedDocs = [], multi
;
if (typeof options === 'function') { cb = options; options = {}; }
callback = cb || function () {};
multi = options.multi !== undefined ? options.multi : false;
this.getCandidates(query, true, function (err, candidates) {
if (err) { return callback(err); }
try {
candidates.forEach(function (d) {
if (model.match(d, query) && (multi || numRemoved === 0)) {
numRemoved += 1;
removedDocs.push({ $$deleted: true, _id: d._id });
self.removeFromIndexes(d);
}
});
} catch (err) { return callback(err); }
self.persistence.persistNewState(removedDocs, function (err) {
if (err) { return callback(err); }
return callback(null, numRemoved);
});
});
}...
valid = false;
}
});
if (valid) { validDocs.push(doc); } else { expiredDocsIds.push(doc._id); }
});
async.eachSeries(expiredDocsIds, function (_id, cb) {
self._remove({ _id: _id }, {}, function (err) {
if (err) { return callback(err); }
return cb();
});
}, function (err) {
return callback(null, validDocs);
});
}]);
..._update = function (query, updateQuery, options, cb) {
var callback
, self = this
, numReplaced = 0
, multi, upsert
, i
;
if (typeof options === 'function') { cb = options; options = {}; }
callback = cb || function () {};
multi = options.multi !== undefined ? options.multi : false;
upsert = options.upsert !== undefined ? options.upsert : false;
async.waterfall([
function (cb) { // If upsert option is set, check whether we need to insert the doc
if (!upsert) { return cb(); }
// Need to use an internal function not tied to the executor to avoid deadlock
var cursor = new Cursor(self, query);
cursor.limit(1)._exec(function (err, docs) {
if (err) { return callback(err); }
if (docs.length === 1) {
return cb();
} else {
var toBeInserted;
try {
model.checkObject(updateQuery);
// updateQuery is a simple object with no modifier, use it as the document to insert
toBeInserted = updateQuery;
} catch (e) {
// updateQuery contains modifiers, use the find query as the base,
// strip it from all operators and update it according to updateQuery
try {
toBeInserted = model.modify(model.deepCopy(query, true), updateQuery);
} catch (err) {
return callback(err);
}
}
return self._insert(toBeInserted, function (err, newDoc) {
if (err) { return callback(err); }
return callback(null, 1, newDoc, true);
});
}
});
}
, function () { // Perform the update
var modifiedDoc , modifications = [], createdAt;
self.getCandidates(query, function (err, candidates) {
if (err) { return callback(err); }
// Preparing update (if an error is thrown here neither the datafile nor
// the in-memory indexes are affected)
try {
for (i = 0; i < candidates.length; i += 1) {
if (model.match(candidates[i], query) && (multi || numReplaced === 0)) {
numReplaced += 1;
if (self.timestampData) { createdAt = candidates[i].createdAt; }
modifiedDoc = model.modify(candidates[i], updateQuery);
if (self.timestampData) {
modifiedDoc.createdAt = createdAt;
modifiedDoc.updatedAt = new Date();
}
modifications.push({ oldDoc: candidates[i], newDoc: modifiedDoc });
}
}
} catch (err) {
return callback(err);
}
// Change the docs in memory
try {
self.updateIndexes(modifications);
} catch (err) {
return callback(err);
}
// Update the datafile
var updatedDocs = _.pluck(modifications, 'newDoc');
self.persistence.persistNewState(updatedDocs, function (err) {
if (err) { return callback(err); }
if (!options.returnUpdatedDocs) {
return callback(null, numReplaced);
} else {
var updatedDocsDC = [];
updatedDocs.forEach(function (doc) { updatedDocsDC.push(model.deepCopy(doc)); });
if (! multi) { updatedDocsDC = updatedDocsDC[0]; }
return callback(null, numReplaced, updatedDocsDC);
}
});
});
}]);
}n/a
addToIndexes = function (doc) {
var i, failingIndex, error
, keys = Object.keys(this.indexes)
;
for (i = 0; i < keys.length; i += 1) {
try {
this.indexes[keys[i]].insert(doc);
} catch (e) {
failingIndex = i;
error = e;
break;
}
}
// If an error happened, we need to rollback the insert on all other indexes
if (error) {
for (i = 0; i < failingIndex; i += 1) {
this.indexes[keys[i]].remove(doc);
}
throw error;
}
}...
* If newDoc is an array of documents, this will insert all documents in the cache
* @api private
*/
Datastore.prototype._insertInCache = function (preparedDoc) {
if (util.isArray(preparedDoc)) {
this._insertMultipleDocsInCache(preparedDoc);
} else {
this.addToIndexes(preparedDoc);
}
};
/**
* If one insertion fails (e.g. because of a unique constraint), roll back all previous
* inserts and throws the error
* @api private
...count = function (query, callback) {
var cursor = new Cursor(this, query, function(err, docs, callback) {
if (err) { return callback(err); }
return callback(null, docs.length);
});
if (typeof callback === 'function') {
cursor.exec(callback);
} else {
return cursor;
}
}...
### Counting documents
You can use `count` to count documents. It has the same syntax as `find`. For example:
```javascript
// Count all planets in the solar system
db.count({ system: 'solar' }, function (err, count) {
// count equals to 3
});
// Count all documents in the datastore
db.count({}, function (err, count) {
// count equals to 4
});
...createNewId = function () {
var tentativeId = customUtils.uid(16);
// Try as many times as needed to get an unused _id. As explained in customUtils, the probability of this ever happening is extremely
small, so this is O(1)
if (this.indexes._id.getMatching(tentativeId).length > 0) {
tentativeId = this.createNewId();
}
return tentativeId;
}...
/**
* Create a new _id that's not already in use
*/
Datastore.prototype.createNewId = function () {
var tentativeId = customUtils.uid(16);
// Try as many times as needed to get an unused _id. As explained in customUtils, the probability of this ever happening is extremely
small, so this is O(1)
if (this.indexes._id.getMatching(tentativeId).length > 0) {
tentativeId = this.createNewId();
}
return tentativeId;
};
/**
* Prepare a document (or array of documents) to be inserted in a database
* Meaning adds _id and timestamps if necessary on a copy of newDoc to avoid any side effect on user input
...ensureIndex = function (options, cb) {
var err
, callback = cb || function () {};
options = options || {};
if (!options.fieldName) {
err = new Error("Cannot create an index without a fieldName");
err.missingFieldName = true;
return callback(err);
}
if (this.indexes[options.fieldName]) { return callback(null); }
this.indexes[options.fieldName] = new Index(options);
if (options.expireAfterSeconds !== undefined) { this.ttlIndexes[options.fieldName] = options.expireAfterSeconds; } // With this
implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here
try {
this.indexes[options.fieldName].insert(this.getAllData());
} catch (e) {
delete this.indexes[options.fieldName];
return callback(e);
}
// We may want to force all options to be persisted including defaults, not just the ones passed the index creation function
this.persistence.persistNewState([{ $$indexCreated: options }], function (err) {
if (err) { return callback(err); }
return callback(null);
});
}...
db.remove({}, { multi: true }, function (err, numRemoved) {
});
```
### Indexing
NeDB supports indexing. It gives a very nice speed boost and can be used to enforce a unique constraint on a field. You can index
any field, including fields in nested documents using the dot notation. For now, indexes are only used to speed up basic queries
and queries using `$in`, `$lt`, `$lte`, `$gt` and `$gte`. The indexed values cannot be of type array of object.
To create an index, use `datastore.ensureIndex(options, cb)`, where callback is optional
and get passed an error if any (usually a unique constraint that was violated). `ensureIndex` can be called when you want, even
after some data was inserted, though it's best to call it at application startup. The options are:
* **fieldName** (required): name of the field to index. Use the dot notation to index a field in a nested document.
* **unique** (optional, defaults to `false`): enforce field uniqueness. Note that a unique index will raise an error if you try
to index two documents for which the field is not defined.
* **sparse** (optional, defaults to `false`): don't index documents for which the field is not defined. Use this option along
with "unique" if you want to accept multiple documents for which it is not defined.
* **expireAfterSeconds** (number of seconds, optional): if set, the created index is a TTL (time to live) index, that will automatically
remove documents when the system date becomes larger than the date on the indexed field plus `expireAfterSeconds`. Documents where
the indexed field is not specified or not a `Date` object are ignored
Note: the `_id` is automatically indexed with a unique constraint, no need to call `ensureIndex` on it.
...find = function (query, projection, callback) {
switch (arguments.length) {
case 1:
projection = {};
// callback is undefined, will return a cursor
break;
case 2:
if (typeof projection === 'function') {
callback = projection;
projection = {};
} // If not assume projection is an object and callback undefined
break;
}
var cursor = new Cursor(this, query, function(err, docs, callback) {
var res = [], i;
if (err) { return callback(err); }
for (i = 0; i < docs.length; i += 1) {
res.push(model.deepCopy(docs[i]));
}
return callback(null, res);
});
cursor.projection(projection);
if (typeof callback === 'function') {
cursor.exec(callback);
} else {
return cursor;
}
}...
// { _id: 'id1', planet: 'Mars', system: 'solar', inhabited: false, satellites: ['Phobos'
;, 'Deimos'] }
// { _id: 'id2', planet: 'Earth', system: 'solar', inhabited: true, humans: { genders: 2, eyes: true
} }
// { _id: 'id3', planet: 'Jupiter', system: 'solar', inhabited: false }
// { _id: 'id4', planet: 'Omicron Persei 8', system: 'futurama', inhabited: true, humans: { genders
: 7 } }
// { _id: 'id5', completeData: { planets: [ { name: 'Earth', number: 3 }, { name: 'Mars', number:
2 }, { name: 'Pluton', number: 9 } ] } }
// Finding all planets in the solar system
db.find({ system: 'solar' }, function (err, docs) {
// docs is an array containing documents Mars, Earth, Jupiter
// If no document is found, docs is equal to []
});
// Finding all planets whose name contain the substring 'ar' using a regular expression
db.find({ planet: /ar/ }, function (err, docs) {
// docs contains Mars and Earth
...findOne = function (query, projection, callback) {
switch (arguments.length) {
case 1:
projection = {};
// callback is undefined, will return a cursor
break;
case 2:
if (typeof projection === 'function') {
callback = projection;
projection = {};
} // If not assume projection is an object and callback undefined
break;
}
var cursor = new Cursor(this, query, function(err, docs, callback) {
if (err) { return callback(err); }
if (docs.length === 1) {
return callback(null, model.deepCopy(docs[0]));
} else {
return callback(null, null);
}
});
cursor.projection(projection).limit(1);
if (typeof callback === 'function') {
cursor.exec(callback);
} else {
return cursor;
}
}...
});
// Find all documents in the collection
db.find({}, function (err, docs) {
});
// The same rules apply when you want to only find one document
db.findOne({ _id: 'id1' }, function (err, doc) {
// doc is the document Mars
// If no document is found, doc is null
});
```
#### Operators ($lt, $lte, $gt, $gte, $in, $nin, $ne, $exists, $regex)
The syntax is `{ field: { $op: value } }` where `$op` is any comparison operator:
...getAllData = function () {
return this.indexes._id.getAll();
}...
}
if (this.indexes[options.fieldName]) { return callback(null); }
this.indexes[options.fieldName] = new Index(options);
if (options.expireAfterSeconds !== undefined) { this.ttlIndexes[options.fieldName] = options.expireAfterSeconds; } // With this
implementation index creation is not necessary to ensure TTL but we stick with MongoDB's API here
try {
this.indexes[options.fieldName].insert(this.getAllData());
} catch (e) {
delete this.indexes[options.fieldName];
return callback(e);
}
// We may want to force all options to be persisted including defaults, not just the ones passed the index creation function
this.persistence.persistNewState([{ $$indexCreated: options }], function (err) {
...getCandidates = function (query, dontExpireStaleDocs, callback) {
var indexNames = Object.keys(this.indexes)
, self = this
, usableQueryKeys;
if (typeof dontExpireStaleDocs === 'function') {
callback = dontExpireStaleDocs;
dontExpireStaleDocs = false;
}
async.waterfall([
// STEP 1: get candidates list by checking indexes from most to least frequent usecase
function (cb) {
// For a basic match
usableQueryKeys = [];
Object.keys(query).forEach(function (k) {
if (typeof query[k] === 'string' || typeof query[k] === 'number' || typeof query[k] === 'boolean' || util.isDate(query[k]) ||
query[k] === null) {
usableQueryKeys.push(k);
}
});
usableQueryKeys = _.intersection(usableQueryKeys, indexNames);
if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]]));
}
// For a $in match
usableQueryKeys = [];
Object.keys(query).forEach(function (k) {
if (query[k] && query[k].hasOwnProperty('$in')) {
usableQueryKeys.push(k);
}
});
usableQueryKeys = _.intersection(usableQueryKeys, indexNames);
if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getMatching(query[usableQueryKeys[0]].$in));
}
// For a comparison match
usableQueryKeys = [];
Object.keys(query).forEach(function (k) {
if (query[k] && (query[k].hasOwnProperty('$lt') || query[k].hasOwnProperty('$lte') || query[k].hasOwnProperty('$gt') || query
[k].hasOwnProperty('$gte'))) {
usableQueryKeys.push(k);
}
});
usableQueryKeys = _.intersection(usableQueryKeys, indexNames);
if (usableQueryKeys.length > 0) {
return cb(null, self.indexes[usableQueryKeys[0]].getBetweenBounds(query[usableQueryKeys[0]]));
}
// By default, return all the DB data
return cb(null, self.getAllData());
}
// STEP 2: remove all expired documents
, function (docs) {
if (dontExpireStaleDocs) { return callback(null, docs); }
var expiredDocsIds = [], validDocs = [], ttlIndexesFieldNames = Object.keys(self.ttlIndexes);
docs.forEach(function (doc) {
var valid = true;
ttlIndexesFieldNames.forEach(function (i) {
if (doc[i] !== undefined && util.isDate(doc[i]) && Date.now() > doc[i].getTime() + self.ttlIndexes[i] * 1000)Â {
valid = false;
}
});
if (valid) { validDocs.push(doc); } else { expiredDocsIds.push(doc._id); }
});
async.eachSeries(expiredDocsIds, function (_id, cb) {
self._remove({ _id: _id }, {}, function (err) {
if (err) { return callback(err); }
return cb();
});
}, function (err) {
return callback(null, validDocs);
});
}]);
}...
if (self.execFn) {
return self.execFn(error, res, _callback);
} else {
return _callback(error, res);
}
}
this.db.getCandidates(this.query, function (err, candidates) {
if (err) { return callback(err); }
try {
for (i = 0; i < candidates.length; i += 1) {
if (model.match(candidates[i], self.query)) {
// If a sort is defined, wait for the results to be sorted before applying limit and skip
if (!self._sort) {
...insert = function () {
this.executor.push({ this: this, fn: this._insert, arguments: arguments });
}...
, nedbIsAwesome: true
, notthere: null
, notToBeSaved: undefined // Will not be saved
, fruits: [ 'apple', 'orange', 'pear' ]
, infos: { name: 'nedb' }
};
db.insert(doc, function (err, newDoc) { // Callback is optional
// newDoc is the newly inserted document, including its _id
// newDoc has no key called notToBeSaved since its value was undefined
});
```
You can also bulk-insert an array of documents. This operation is atomic, meaning that if one insert fails due to a unique constraint
being violated, all changes are rolled back.
...loadDatabase = function () {
this.executor.push({ this: this.persistence, fn: this.persistence.loadDatabase, arguments: arguments }, true);
}...
var Datastore = require('nedb')
, db = new Datastore();
// Type 2: Persistent datastore with manual loading
var Datastore = require('nedb')
, db = new Datastore({ filename: 'path/to/datafile' });
db.loadDatabase(function (err) { // Callback is optional
// Now commands will be executed
});
// Type 3: Persistent datastore with automatic loading
var Datastore = require('nedb')
, db = new Datastore({ filename: 'path/to/datafile', autoload: true });
...prepareDocumentForInsertion = function (newDoc) {
var preparedDoc, self = this;
if (util.isArray(newDoc)) {
preparedDoc = [];
newDoc.forEach(function (doc) { preparedDoc.push(self.prepareDocumentForInsertion(doc)); });
} else {
preparedDoc = model.deepCopy(newDoc);
if (preparedDoc._id === undefined) { preparedDoc._id = this.createNewId(); }
var now = new Date();
if (this.timestampData && preparedDoc.createdAt === undefined) { preparedDoc.createdAt = now; }
if (this.timestampData && preparedDoc.updatedAt === undefined) { preparedDoc.updatedAt = now; }
model.checkObject(preparedDoc);
}
return preparedDoc;
}...
*/
Datastore.prototype._insert = function (newDoc, cb) {
var callback = cb || function () {}
, preparedDoc
;
try {
preparedDoc = this.prepareDocumentForInsertion(newDoc)
this._insertInCache(preparedDoc);
} catch (e) {
return callback(e);
}
this.persistence.persistNewState(util.isArray(preparedDoc) ? preparedDoc : [preparedDoc], function (err) {
if (err) { return callback(err); }
...remove = function () {
this.executor.push({ this: this, fn: this._remove, arguments: arguments });
}...
db.update({ _id: 'id1' }, { $min: { value: 8 } }, {}, function () {
// The document will not be modified
});
```
### Removing documents
`db.remove(query, options, callback)` will remove all documents matching `query` according
to `options`
* `query` is the same as the ones used for finding and updating
* `options` only one option for now: `multi` which allows the removal of multiple documents if set to true. Default is false
* `callback` is optional, signature: err, numRemoved
```javascript
// Let's use the same example collection as in the "finding document" part
// { _id: 'id1', planet: 'Mars', system: 'solar', inhabited: false }
...removeFromIndexes = function (doc) {
var self = this;
Object.keys(this.indexes).forEach(function (i) {
self.indexes[i].remove(doc);
});
}...
failingI = i;
break;
}
}
if (error) {
for (i = 0; i < failingI; i += 1) {
this.removeFromIndexes(preparedDocs[i]);
}
throw error;
}
};
Datastore.prototype.insert = function () {
...removeIndex = function (fieldName, cb) {
var callback = cb || function () {};
delete this.indexes[fieldName];
this.persistence.persistNewState([{ $$indexRemoved: fieldName }], function (err) {
if (err) { return callback(err); }
return callback(null);
});
}...
* **fieldName** (required): name of the field to index. Use the dot notation to index a field in a nested document.
* **unique** (optional, defaults to `false`): enforce field uniqueness. Note that a unique index will raise an error if you try
to index two documents for which the field is not defined.
* **sparse** (optional, defaults to `false`): don't index documents for which the field is not defined. Use this option along
with "unique" if you want to accept multiple documents for which it is not defined.
* **expireAfterSeconds** (number of seconds, optional): if set, the created index is a TTL (time to live) index, that will automatically
remove documents when the system date becomes larger than the date on the indexed field plus `expireAfterSeconds`. Documents where
the indexed field is not specified or not a `Date` object are ignored
Note: the `_id` is automatically indexed with a unique constraint, no need to call `ensureIndex` on it.
You can remove a previously created index with `datastore.removeIndex(fieldName, cb)`.
If your datastore is persistent, the indexes you created are persisted in the datafile, when you load the database a second time
they are automatically created for you. No need to remove any `ensureIndex` though, if it is called on a database that already
has the index, nothing happens.
```javascript
db.ensureIndex({ fieldName: 'somefield' }, function (err) {
// If there was an error, err is not null
});
...resetIndexes = function (newData) {
var self = this;
Object.keys(this.indexes).forEach(function (i) {
self.indexes[i].reset(newData);
});
}...
* @param {Function} cb Optional callback, signature: err
*/
Persistence.prototype.loadDatabase = function (cb) {
var callback = cb || function () {}
, self = this
;
self.db.resetIndexes();
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
...update = function () {
this.executor.push({ this: this, fn: this._update, arguments: arguments });
}...
db.count({}, function (err, count) {
// count equals to 4
});
```
### Updating documents
`db.update(query, update, options, callback)` will update all documents matching `query
` according to the `update` rules:
* `query` is the same kind of finding query you use with `find` and `findOne`
* `update` specifies how the documents should be modified. It is either a new document or a set of modifiers (you cannot use both
together, it doesn't make sense!)
* A new document will replace the matched docs
* The modifiers create the fields they need to modify if they don't exist, and you can apply them to subdocs. Available field
modifiers are `$set` to change a field's value, `$unset` to delete a field, `$inc` to increment a field's value and `$
min`/`$max` to change field's value, only if provided value is less/greater than current value. To work on arrays, you have
`$push`, `$pop`, `$addToSet`, `$pull`, and the special `$each` and `$slice`. See examples below for the syntax.
* `options` is an object with two possible parameters
* `multi` (defaults to `false`) which allows the modification of several documents if set to true
* `upsert` (defaults to `false`) if you want to insert a new document corresponding to the `update` rules if your `query` doesn&#
x27;t match anything. If your `update` is a simple object with no modifiers, it is the inserted document. In the other case, the
`query` is stripped from all operator recursively, and the `update` is applied to it.
...updateIndexes = function (oldDoc, newDoc) {
var i, failingIndex, error
, keys = Object.keys(this.indexes)
;
for (i = 0; i < keys.length; i += 1) {
try {
this.indexes[keys[i]].update(oldDoc, newDoc);
} catch (e) {
failingIndex = i;
error = e;
break;
}
}
// If an error happened, we need to rollback the update on all other indexes
if (error) {
for (i = 0; i < failingIndex; i += 1) {
this.indexes[keys[i]].revertUpdate(oldDoc, newDoc);
}
throw error;
}
}...
}
} catch (err) {
return callback(err);
}
// Change the docs in memory
try {
self.updateIndexes(modifications);
} catch (err) {
return callback(err);
}
// Update the datafile
var updatedDocs = _.pluck(modifications, 'newDoc');
self.persistence.persistNewState(updatedDocs, function (err) {
...function Executor() {
this.buffer = [];
this.ready = false;
// This queue will execute all commands, one-by-one in order
this.queue = async.queue(function (task, cb) {
var newArguments = [];
// task.arguments is an array-like object on which adding a new field doesn't work, so we transform it into a real array
for (var i = 0; i < task.arguments.length; i += 1) { newArguments.push(task.arguments[i]); }
var lastArg = task.arguments[task.arguments.length - 1];
// Always tell the queue task is complete. Execute callback if any was given.
if (typeof lastArg === 'function') {
// Callback was supplied
newArguments[newArguments.length - 1] = function () {
if (typeof setImmediate === 'function') {
setImmediate(cb);
} else {
process.nextTick(cb);
}
lastArg.apply(null, arguments);
};
} else if (!lastArg && task.arguments.length !== 0) {
// false/undefined/null supplied as callbback
newArguments[newArguments.length - 1] = function () { cb(); };
} else {
// Nothing supplied as callback
newArguments.push(function () { cb(); });
}
task.fn.apply(task.this, newArguments);
}, 1);
}n/a
processBuffer = function () {
var i;
this.ready = true;
for (i = 0; i < this.buffer.length; i += 1) { this.queue.push(this.buffer[i]); }
this.buffer = [];
}...
});
});
});
}
], function (err) {
if (err) { return callback(err); }
self.db.executor.processBuffer();
return callback(null);
});
};
// Interface
module.exports = Persistence;
...push = function (task, forceQueuing) {
if (this.ready || forceQueuing) {
this.queue.push(task);
} else {
this.buffer.push(task);
}
}...
;
// Preparing all the $in arrays, will take some time
for (i = 0; i < n; i += 1) {
ins[i] = [];
for (j = 0; j < arraySize; j += 1) {
ins[i].push((i + j) % n);
}
}
profiler.step("Finding " + n + " documents WITH $IN OPERATOR");
function runFrom(i) {
if (i === n) { // Finished
...function areThingsEqual(a, b) {
var aKeys , bKeys , i;
// Strings, booleans, numbers, null
if (a === null || typeof a === 'string' || typeof a === 'boolean' || typeof a === 'number' ||
b === null || typeof b === 'string' || typeof b === 'boolean' || typeof b === 'number') { return a === b; }
// Dates
if (util.isDate(a) || util.isDate(b)) { return util.isDate(a) && util.isDate(b) && a.getTime() === b.getTime(); }
// Arrays (no match since arrays are used as a $in)
// undefined (no match since they mean field doesn't exist and can't be serialized)
if ((!(util.isArray(a) && util.isArray(b)) && (util.isArray(a) || util.isArray(b))) || a === undefined || b === undefined) { return
false; }
// General objects (check for deep equality)
// a and b should be objects at this point
try {
aKeys = Object.keys(a);
bKeys = Object.keys(b);
} catch (e) {
return false;
}
if (aKeys.length !== bKeys.length) { return false; }
for (i = 0; i < aKeys.length; i += 1) {
if (bKeys.indexOf(aKeys[i]) === -1) { return false; }
if (!areThingsEqual(a[aKeys[i]], b[aKeys[i]])) { return false; }
}
return true;
}...
var toTest = [null, 'somestring', 42, true, new Date(72998322), { hello: 'world' }]
, toTestAgainst = [null, 'somestring', 42, true, new Date(72998322), { hello: 'world' }] // Use another
array so that we don't test pointer equality
, i, j
;
for (i = 0; i < toTest.length; i += 1) {
for (j = 0; j < toTestAgainst.length; j += 1) {
model.areThingsEqual(toTest[i], toTestAgainst[j]).should.equal(i === j);
}
}
});
it('Can test native types null undefined string number boolean date equality', function () {
var toTest = [null, undefined, 'somestring', 42, true, new Date(72998322), { hello: 'world' }]
, toTestAgainst = [undefined, null, 'someotherstring', 5, false, new Date(111111), { hello: 'mars' }]
...function checkObject(obj) {
if (util.isArray(obj)) {
obj.forEach(function (o) {
checkObject(o);
});
}
if (typeof obj === 'object' && obj !== null) {
Object.keys(obj).forEach(function (k) {
checkKey(k, obj[k]);
checkObject(obj[k]);
});
}
}...
newDoc.forEach(function (doc) { preparedDoc.push(self.prepareDocumentForInsertion(doc)); });
} else {
preparedDoc = model.deepCopy(newDoc);
if (preparedDoc._id === undefined) { preparedDoc._id = this.createNewId(); }
var now = new Date();
if (this.timestampData && preparedDoc.createdAt === undefined) { preparedDoc.createdAt = now; }
if (this.timestampData && preparedDoc.updatedAt === undefined) { preparedDoc.updatedAt = now; }
model.checkObject(preparedDoc);
}
return preparedDoc;
};
/**
* If newDoc is an array of documents, this will insert all documents in the cache
...function compareThings(a, b, _compareStrings) {
var aKeys, bKeys, comp, i
, compareStrings = _compareStrings || compareNSB;
// undefined
if (a === undefined) { return b === undefined ? 0 : -1; }
if (b === undefined) { return a === undefined ? 0 : 1; }
// null
if (a === null) { return b === null ? 0 : -1; }
if (b === null) { return a === null ? 0 : 1; }
// Numbers
if (typeof a === 'number') { return typeof b === 'number' ? compareNSB(a, b) : -1; }
if (typeof b === 'number') { return typeof a === 'number' ? compareNSB(a, b) : 1; }
// Strings
if (typeof a === 'string') { return typeof b === 'string' ? compareStrings(a, b) : -1; }
if (typeof b === 'string') { return typeof a === 'string' ? compareStrings(a, b) : 1; }
// Booleans
if (typeof a === 'boolean') { return typeof b === 'boolean' ? compareNSB(a, b) : -1; }
if (typeof b === 'boolean') { return typeof a === 'boolean' ? compareNSB(a, b) : 1; }
// Dates
if (util.isDate(a)) { return util.isDate(b) ? compareNSB(a.getTime(), b.getTime()) : -1; }
if (util.isDate(b)) { return util.isDate(a) ? compareNSB(a.getTime(), b.getTime()) : 1; }
// Arrays (first element is most significant and so on)
if (util.isArray(a)) { return util.isArray(b) ? compareArrays(a, b) : -1; }
if (util.isArray(b)) { return util.isArray(a) ? compareArrays(a, b) : 1; }
// Objects
aKeys = Object.keys(a).sort();
bKeys = Object.keys(b).sort();
for (i = 0; i < Math.min(aKeys.length, bKeys.length); i += 1) {
comp = compareThings(a[aKeys[i]], b[bKeys[i]]);
if (comp !== 0) { return comp; }
}
return compareNSB(aKeys.length, bKeys.length);
}...
key = keys[i];
criteria.push({ key: key, direction: self._sort[key] });
}
res.sort(function(a, b) {
var criterion, compare, i;
for (i = 0; i < criteria.length; i++) {
criterion = criteria[i];
compare = criterion.direction * model.compareThings(model.getDotValue(a, criterion
.key), model.getDotValue(b, criterion.key), self.db.compareStrings);
if (compare !== 0) {
return compare;
}
}
return 0;
});
...function deepCopy(obj, strictKeys) {
var res;
if ( typeof obj === 'boolean' ||
typeof obj === 'number' ||
typeof obj === 'string' ||
obj === null ||
(util.isDate(obj)) ) {
return obj;
}
if (util.isArray(obj)) {
res = [];
obj.forEach(function (o) { res.push(deepCopy(o, strictKeys)); });
return res;
}
if (typeof obj === 'object') {
res = {};
Object.keys(obj).forEach(function (k) {
if (!strictKeys || (k[0] !== '$' && k.indexOf('.') === -1)) {
res[k] = deepCopy(obj[k], strictKeys);
}
});
return res;
}
return undefined; // For now everything else is undefined. We should probably throw an error instead
}...
this._insertInCache(preparedDoc);
} catch (e) {
return callback(e);
}
this.persistence.persistNewState(util.isArray(preparedDoc) ? preparedDoc : [preparedDoc], function (err) {
if (err) { return callback(err); }
return callback(null, model.deepCopy(preparedDoc));
});
};
/**
* Create a new _id that's not already in use
*/
Datastore.prototype.createNewId = function () {
...function deserialize(rawData) {
return JSON.parse(rawData, function (k, v) {
if (k === '$$date') { return new Date(v); }
if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean' || v === null) { return v; }
if (v && v.$$date) { return v.$$date; }
return v;
});
}...
, corruptItems = -1 // Last line of every data file is usually blank so not really corrupt
;
for (i = 0; i < data.length; i += 1) {
var doc;
try {
doc = model.deserialize(this.beforeDeserialization(data[i]));
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id];
} else {
dataById[doc._id] = doc;
}
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != undefined) {
...function getDotValue(obj, field) {
var fieldParts = typeof field === 'string' ? field.split('.') : field
, i, objs;
if (!obj) { return undefined; } // field cannot be empty so that means we should return undefined so that nothing can match
if (fieldParts.length === 0) { return obj; }
if (fieldParts.length === 1) { return obj[fieldParts[0]]; }
if (util.isArray(obj[fieldParts[0]])) {
// If the next field is an integer, return only this item of the array
i = parseInt(fieldParts[1], 10);
if (typeof i === 'number' && !isNaN(i)) {
return getDotValue(obj[fieldParts[0]][i], fieldParts.slice(2))
}
// Return the array of values
objs = new Array();
for (i = 0; i < obj[fieldParts[0]].length; i += 1) {
objs.push(getDotValue(obj[fieldParts[0]][i], fieldParts.slice(1)));
}
return objs;
} else {
return getDotValue(obj[fieldParts[0]], fieldParts.slice(1));
}
}...
// Do the actual projection
candidates.forEach(function (candidate) {
var toPush;
if (action === 1) { // pick-type projection
toPush = { $set: {} };
keys.forEach(function (k) {
toPush.$set[k] = model.getDotValue(candidate, k);
if (toPush.$set[k] === undefined) { delete toPush.$set[k]; }
});
toPush = model.modify({}, toPush);
} else { // omit-type projection
toPush = { $unset: {} };
keys.forEach(function (k) { toPush.$unset[k] = true });
toPush = model.modify(candidate, toPush);
...function isPrimitiveType(obj) {
return ( typeof obj === 'boolean' ||
typeof obj === 'number' ||
typeof obj === 'string' ||
obj === null ||
util.isDate(obj) ||
util.isArray(obj));
}...
it('Properties with a null value dont trigger an error', function () {
var obj = { prop: null };
model.checkObject(obj);
});
it('Can check if an object is a primitive or not', function () {
model.isPrimitiveType(5).should.equal(true);
model.isPrimitiveType('sdsfdfs').should.equal(true);
model.isPrimitiveType(0).should.equal(true);
model.isPrimitiveType(true).should.equal(true);
model.isPrimitiveType(false).should.equal(true);
model.isPrimitiveType(new Date()).should.equal(true);
model.isPrimitiveType([]).should.equal(true);
model.isPrimitiveType([3, 'try']).should.equal(true);
...function match(obj, query) {
var queryKeys, queryKey, queryValue, i;
// Primitive query against a primitive type
// This is a bit of a hack since we construct an object with an arbitrary key only to dereference it later
// But I don't have time for a cleaner implementation now
if (isPrimitiveType(obj) || isPrimitiveType(query)) {
return matchQueryPart({ needAKey: obj }, 'needAKey', query);
}
// Normal query
queryKeys = Object.keys(query);
for (i = 0; i < queryKeys.length; i += 1) {
queryKey = queryKeys[i];
queryValue = query[queryKey];
if (queryKey[0] === '$') {
if (!logicalOperators[queryKey]) { throw new Error("Unknown logical operator " + queryKey); }
if (!logicalOperators[queryKey](obj, queryValue)) { return false; }
} else {
if (!matchQueryPart(obj, queryKey, queryValue)) { return false; }
}
}
return true;
}...
}
this.db.getCandidates(this.query, function (err, candidates) {
if (err) { return callback(err); }
try {
for (i = 0; i < candidates.length; i += 1) {
if (model.match(candidates[i], self.query)) {
// If a sort is defined, wait for the results to be sorted before applying limit and skip
if (!self._sort) {
if (self._skip && self._skip > skipped) {
skipped += 1;
} else {
res.push(candidates[i]);
added += 1;
...function modify(obj, updateQuery) {
var keys = Object.keys(updateQuery)
, firstChars = _.map(keys, function (item) { return item[0]; })
, dollarFirstChars = _.filter(firstChars, function (c) { return c === '$'; })
, newDoc, modifiers
;
if (keys.indexOf('_id') !== -1 && updateQuery._id !== obj._id) { throw new Error("You cannot change a document's _id"); }
if (dollarFirstChars.length !== 0 && dollarFirstChars.length !== firstChars.length) {
throw new Error("You cannot mix modifiers and normal fields");
}
if (dollarFirstChars.length === 0) {
// Simply replace the object with the update query contents
newDoc = deepCopy(updateQuery);
newDoc._id = obj._id;
} else {
// Apply modifiers
modifiers = _.uniq(keys);
newDoc = deepCopy(obj);
modifiers.forEach(function (m) {
var keys;
if (!modifierFunctions[m]) { throw new Error("Unknown modifier " + m); }
// Can't rely on Object.keys throwing on non objects since ES6
// Not 100% satisfying as non objects can be interpreted as objects but no false negatives so we can live with it
if (typeof updateQuery[m] !== 'object') {
throw new Error("Modifier " + m + "'s argument must be an object");
}
keys = Object.keys(updateQuery[m]);
keys.forEach(function (k) {
modifierFunctions[m](newDoc, k, updateQuery[m][k]);
});
});
}
// Check result is valid and return it
checkObject(newDoc);
if (obj._id !== newDoc._id) { throw new Error("You can't change a document's _id"); }
return newDoc;
}...
var toPush;
if (action === 1) { // pick-type projection
toPush = { $set: {} };
keys.forEach(function (k) {
toPush.$set[k] = model.getDotValue(candidate, k);
if (toPush.$set[k] === undefined) { delete toPush.$set[k]; }
});
toPush = model.modify({}, toPush);
} else { // omit-type projection
toPush = { $unset: {} };
keys.forEach(function (k) { toPush.$unset[k] = true });
toPush = model.modify(candidate, toPush);
}
if (keepId) {
toPush._id = candidate._id;
...function serialize(obj) {
var res;
res = JSON.stringify(obj, function (k, v) {
checkKey(k, v);
if (v === undefined) { return undefined; }
if (v === null) { return null; }
// Hackish way of checking if object is Date (this way it works between execution contexts in node-webkit).
// We can't use value directly because for dates it is already string in this function (date.toJSON was already called), so
we use this
if (typeof this[k].getTime === 'function') { return { $$date: this[k].getTime() }; }
return v;
});
return res;
}...
, toPersist = ''
, self = this
;
if (this.inMemoryOnly) { return callback(null); }
this.db.getAllData().forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
});
Object.keys(this.db.indexes).forEach(function (fieldName) {
if (fieldName != "_id") { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName
].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n';
}
});
...function Persistence(options) {
var i, j, randomString;
this.db = options.db;
this.inMemoryOnly = this.db.inMemoryOnly;
this.filename = this.db.filename;
this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
throw new Error("The datafile name can't end with a ~, which is reserved for crash safe backup files");
}
// After serialization and before deserialization hooks with some basic sanity checks
if (options.afterSerialization && !options.beforeDeserialization) {
throw new Error("Serialization hook defined but deserialization hook undefined, cautiously refusing to start NeDB to prevent
dataloss");
}
if (!options.afterSerialization && options.beforeDeserialization) {
throw new Error("Serialization hook undefined but deserialization hook defined, cautiously refusing to start NeDB to prevent
dataloss");
}
this.afterSerialization = options.afterSerialization || function (s) { return s; };
this.beforeDeserialization = options.beforeDeserialization || function (s) { return s; };
for (i = 1; i < 30; i += 1) {
for (j = 0; j < 10; j += 1) {
randomString = customUtils.uid(i);
if (this.beforeDeserialization(this.afterSerialization(randomString)) !== randomString) {
throw new Error("beforeDeserialization is not the reverse of afterSerialization, cautiously refusing to start NeDB to prevent
dataloss");
}
}
}
// For NW apps, store data in the same directory where NW stores application data
if (this.filename && options.nodeWebkitAppName) {
console.log("==================================================================");
console.log("WARNING: The nodeWebkitAppName option is deprecated");
console.log("To get the path to the directory where Node Webkit stores the data");
console.log("for your app, use the internal nw.gui module like this");
console.log("require('nw.gui').App.dataPath");
console.log("See https://github.com/rogerwang/node-webkit/issues/500");
console.log("==================================================================");
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
}
}n/a
ensureDirectoryExists = function (dir, cb) {
var callback = cb || function () {}
;
storage.mkdirp(dir, function (err) { return callback(err); });
}...
};
/**
* Ensure the workspace exists and the db datafile is empty
*/
module.exports.prepareDb = function (filename, cb) {
Persistence.ensureDirectoryExists(path.dirname(filename), function () {
fs.exists(filename, function (exists) {
if (exists) {
fs.unlink(filename, cb);
} else { return cb(); }
});
});
};
...getNWAppFilename = function (appName, relativeFilename) {
var home;
switch (process.platform) {
case 'win32':
case 'win64':
home = process.env.LOCALAPPDATA || process.env.APPDATA;
if (!home) { throw new Error("Couldn't find the base application data folder"); }
home = path.join(home, appName);
break;
case 'darwin':
home = process.env.HOME;
if (!home) { throw new Error("Couldn't find the base application data directory"); }
home = path.join(home, 'Library', 'Application Support', appName);
break;
case 'linux':
home = process.env.HOME;
if (!home) { throw new Error("Couldn't find the base application data directory"); }
home = path.join(home, '.config', appName);
break;
default:
throw new Error("Can't use the Node Webkit relative path for platform " + process.platform);
break;
}
return path.join(home, 'nedb-data', relativeFilename);
}...
console.log("==================================================================");
console.log("WARNING: The nodeWebkitAppName option is deprecated");
console.log("To get the path to the directory where Node Webkit stores the data");
console.log("for your app, use the internal nw.gui module like this");
console.log("require('nw.gui').App.dataPath");
console.log("See https://github.com/rogerwang/node-webkit/issues/500");
console.log("==================================================================");
this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename
);
}
};
/**
* Check if a directory exists and create it on the fly if it is not the case
* cb is optional, signature: err
...compactDatafile = function () {
this.db.executor.push({ this: this, fn: this.persistCachedDatabase, arguments: [] });
}...
, minInterval = 5000
, realInterval = Math.max(interval || 0, minInterval)
;
this.stopAutocompaction();
this.autocompactionIntervalId = setInterval(function () {
self.compactDatafile();
}, realInterval);
};
/**
* Stop autocompaction (do nothing if autocompaction was not running)
*/
...loadDatabase = function (cb) {
var callback = cb || function () {}
, self = this
;
self.db.resetIndexes();
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
storage.ensureDatafileIntegrity(self.filename, function (err) {
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
try {
var treatedData = self.treatRawData(rawData);
} catch (e) {
return cb(e);
}
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) {
self.db.indexes[key] = new Index(treatedData.indexes[key]);
});
// Fill cached database (i.e. all indexes) with data
try {
self.db.resetIndexes(treatedData.data);
} catch (e) {
self.db.resetIndexes(); // Rollback any index which didn't fail
return cb(e);
}
self.db.persistence.persistCachedDatabase(cb);
});
});
});
}
], function (err) {
if (err) { return callback(err); }
self.db.executor.processBuffer();
return callback(null);
});
}...
var Datastore = require('nedb')
, db = new Datastore();
// Type 2: Persistent datastore with manual loading
var Datastore = require('nedb')
, db = new Datastore({ filename: 'path/to/datafile' });
db.loadDatabase(function (err) { // Callback is optional
// Now commands will be executed
});
// Type 3: Persistent datastore with automatic loading
var Datastore = require('nedb')
, db = new Datastore({ filename: 'path/to/datafile', autoload: true });
...persistCachedDatabase = function (cb) {
var callback = cb || function () {}
, toPersist = ''
, self = this
;
if (this.inMemoryOnly) { return callback(null); }
this.db.getAllData().forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
});
Object.keys(this.db.indexes).forEach(function (fieldName) {
if (fieldName != "_id") { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName
].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n';
}
});
storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
if (err) { return callback(err); }
self.db.emit('compaction.done');
return callback(null);
});
}...
try {
self.db.resetIndexes(treatedData.data);
} catch (e) {
self.db.resetIndexes(); // Rollback any index which didn't fail
return cb(e);
}
self.db.persistence.persistCachedDatabase(cb);
});
});
});
}
], function (err) {
if (err) { return callback(err); }
...persistNewState = function (newDocs, cb) {
var self = this
, toPersist = ''
, callback = cb || function () {}
;
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
newDocs.forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
});
if (toPersist.length === 0) { return callback(null); }
storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
return callback(err);
});
}...
this.indexes[options.fieldName].insert(this.getAllData());
} catch (e) {
delete this.indexes[options.fieldName];
return callback(e);
}
// We may want to force all options to be persisted including defaults, not just the ones passed the index creation function
this.persistence.persistNewState([{ $$indexCreated: options }], function (err) {
if (err) { return callback(err); }
return callback(null);
});
};
/**
...setAutocompactionInterval = function (interval) {
var self = this
, minInterval = 5000
, realInterval = Math.max(interval || 0, minInterval)
;
this.stopAutocompaction();
this.autocompactionIntervalId = setInterval(function () {
self.compactDatafile();
}, realInterval);
}...
```
### Persistence
Under the hood, NeDB's persistence uses an append-only format, meaning that all updates and deletes actually result in lines
added at the end of the datafile, for performance reasons. The database is automatically compacted (i.e. put back in the one-line
-per-document format) every time you load each database within your application.
You can manually call the compaction function with `yourDatabase.persistence.compactDatafile` which takes no argument. It queues
a compaction of the datafile in the executor, to be executed sequentially after all pending operations. The datastore will fire
a `compaction.done` event once compaction is finished.
You can also set automatic compaction at regular intervals with `yourDatabase.persistence.setAutocompactionInterval
(interval)`, `interval` in milliseconds (a minimum of 5s is enforced), and stop automatic compaction with `yourDatabase.persistence
.stopAutocompaction()`.
Keep in mind that compaction takes a bit of time (not too much: 130ms for 50k records on a typical development machine) and no other
operation can happen when it does, so most projects actually don't need to use it.
Compaction will also immediately remove any documents whose data line has become corrupted, assuming that the total percentage of
all corrupted documents in that database still falls below the specified `corruptAlertThreshold` option's value.
Durability works similarly to major databases: compaction forces the OS to physically flush data to disk, while appends to the data
file do not (the OS is responsible for flushing the data). That guarantees that a server crash can never cause complete data loss
, while preserving performance. The worst that can happen is a crash between two syncs, causing a loss of all data between the two
syncs. Usually syncs are 30 seconds appart so that's at most 30 seconds of data. <a href="http://oldblog.antirez
.com/post/redis-persistence-demystified.html" target="_blank">This post by Antirez on Redis persistence<
;/a> explains this in more details, NeDB being very close to Redis AOF persistence with `appendfsync` option set to `no`.
...stopAutocompaction = function () {
if (this.autocompactionIntervalId) { clearInterval(this.autocompactionIntervalId); }
}...
```
### Persistence
Under the hood, NeDB's persistence uses an append-only format, meaning that all updates and deletes actually result in lines
added at the end of the datafile, for performance reasons. The database is automatically compacted (i.e. put back in the one-line
-per-document format) every time you load each database within your application.
You can manually call the compaction function with `yourDatabase.persistence.compactDatafile` which takes no argument. It queues
a compaction of the datafile in the executor, to be executed sequentially after all pending operations. The datastore will fire
a `compaction.done` event once compaction is finished.
You can also set automatic compaction at regular intervals with `yourDatabase.persistence.setAutocompactionInterval(interval)`, `
interval` in milliseconds (a minimum of 5s is enforced), and stop automatic compaction with `yourDatabase.persistence.stopAutocompaction()`.
Keep in mind that compaction takes a bit of time (not too much: 130ms for 50k records on a typical development machine) and no other
operation can happen when it does, so most projects actually don't need to use it.
Compaction will also immediately remove any documents whose data line has become corrupted, assuming that the total percentage of
all corrupted documents in that database still falls below the specified `corruptAlertThreshold` option's value.
Durability works similarly to major databases: compaction forces the OS to physically flush data to disk, while appends to the data
file do not (the OS is responsible for flushing the data). That guarantees that a server crash can never cause complete data loss
, while preserving performance. The worst that can happen is a crash between two syncs, causing a loss of all data between the two
syncs. Usually syncs are 30 seconds appart so that's at most 30 seconds of data. <a href="http://oldblog.antirez
.com/post/redis-persistence-demystified.html" target="_blank">This post by Antirez on Redis persistence<
;/a> explains this in more details, NeDB being very close to Redis AOF persistence with `appendfsync` option set to `no`.
...treatRawData = function (rawData) {
var data = rawData.split('\n')
, dataById = {}
, tdata = []
, i
, indexes = {}
, corruptItems = -1 // Last line of every data file is usually blank so not really corrupt
;
for (i = 0; i < data.length; i += 1) {
var doc;
try {
doc = model.deserialize(this.beforeDeserialization(data[i]));
if (doc._id) {
if (doc.$$deleted === true) {
delete dataById[doc._id];
} else {
dataById[doc._id] = doc;
}
} else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != undefined) {
indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated;
} else if (typeof doc.$$indexRemoved === "string") {
delete indexes[doc.$$indexRemoved];
}
} catch (e) {
corruptItems += 1;
}
}
// A bit lenient on corruption
if (data.length > 0 && corruptItems / data.length > this.corruptAlertThreshold) {
throw new Error("More than " + Math.floor(100 * this.corruptAlertThreshold) + "% of the data file is corrupt, the wrong beforeDeserialization
hook may be used. Cautiously refusing to start NeDB to prevent dataloss");
}
Object.keys(dataById).forEach(function (k) {
tdata.push(dataById[k]);
});
return { data: tdata, indexes: indexes };
}...
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
storage.ensureDatafileIntegrity(self.filename, function (err) {
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
try {
var treatedData = self.treatRawData(rawData);
} catch (e) {
return cb(e);
}
// Recreate all indexes in the datafile
Object.keys(treatedData.indexes).forEach(function (key) {
self.db.indexes[key] = new Index(treatedData.indexes[key]);
...appendFile = function (path, data, options, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]);
if (!options || typeof options === 'function') {
options = { encoding: 'utf8', mode: 0o666, flag: 'a' };
} else if (typeof options === 'string') {
options = { encoding: options, mode: 0o666, flag: 'a' };
} else if (typeof options !== 'object') {
throwOptionsError(options);
}
if (!options.flag)
options = util._extend({ flag: 'a' }, options);
// force append behavior when using a supplied file descriptor
if (isFd(path))
options.flag = 'a';
fs.writeFile(path, data, options, callback);
}...
newDocs.forEach(function (doc) {
toPersist += self.afterSerialization(model.serialize(doc)) + '\n';
});
if (toPersist.length === 0) { return callback(null); }
storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
return callback(err);
});
};
/**
* From a database's raw data, return the corresponding
...crashSafeWriteFile = function (filename, data, cb) {
var callback = cb || function () {}
, tempFilename = filename + '~';
async.waterfall([
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
, function (cb) {
storage.exists(filename, function (exists) {
if (exists) {
storage.flushToStorage(filename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
, function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); });
}
, async.apply(storage.flushToStorage, tempFilename)
, function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err); });
}
, async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
], function (err) { return callback(err); })
}...
});
Object.keys(this.db.indexes).forEach(function (fieldName) {
if (fieldName != "_id") { // The special _id index is managed by datastore.js, the others need to be persisted
toPersist += self.afterSerialization(model.serialize({ $$indexCreated: { fieldName: fieldName, unique: self.db.indexes[fieldName
].unique, sparse: self.db.indexes[fieldName].sparse }})) + '\n';
}
});
storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
if (err) { return callback(err); }
self.db.emit('compaction.done');
return callback(null);
});
};
...ensureDatafileIntegrity = function (filename, callback) {
var tempFilename = filename + '~';
storage.exists(filename, function (filenameExists) {
// Write was successful
if (filenameExists) { return callback(null); }
storage.exists(tempFilename, function (oldFilenameExists) {
// New database
if (!oldFilenameExists) {
return storage.writeFile(filename, '', 'utf8', function (err) { callback(err); });
}
// Write failed, use old version
storage.rename(tempFilename, filename, function (err) { return callback(err); });
});
});
}...
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
storage.ensureDatafileIntegrity(self.filename, function (err) {
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
try {
var treatedData = self.treatRawData(rawData);
} catch (e) {
return cb(e);
...ensureFileDoesntExist = function (file, callback) {
storage.exists(file, function (exists) {
if (!exists) { return callback(null); }
storage.unlink(file, function (err) { return callback(err); });
});
}...
describe('Serialization hooks', function () {
var as = function (s) { return "before_" + s + "_after"; }
, bd = function (s) { return s.substring(7, s.length - 6); }
it("Declaring only one hook will throw an exception to prevent data loss", function (done) {
var hookTestFilename = 'workspace/hookTest.db'
storage.ensureFileDoesntExist(hookTestFilename, function () {
fs.writeFileSync(hookTestFilename, "Some content", "utf8");
(function () {
new Datastore({ filename: hookTestFilename, autoload: true
, afterSerialization: as
});
}).should.throw();
...exists = function (path, callback) {
if (!nullCheck(path, cb)) return;
var req = new FSReqWrap();
req.oncomplete = cb;
binding.stat(pathModule._makeLong(path), req);
function cb(err, stats) {
if (callback) callback(err ? false : true);
}
}...
/**
* Ensure the workspace exists and the db datafile is empty
*/
module.exports.prepareDb = function (filename, cb) {
Persistence.ensureDirectoryExists(path.dirname(filename), function () {
fs.exists(filename, function (exists) {
if (exists) {
fs.unlink(filename, cb);
} else { return cb(); }
});
});
};
...flushToStorage = function (options, callback) {
var filename, flags;
if (typeof options === 'string') {
filename = options;
flags = 'r+';
} else {
filename = options.filename;
flags = options.isDir ? 'r' : 'r+';
}
// Windows can't fsync (FlushFileBuffers) directories. We can live with this as it cannot cause 100% dataloss
// except in the very rare event of the first time database is loaded and a crash happens
if (flags === 'r' && (process.platform === 'win32' || process.platform === 'win64')) { return callback(null); }
fs.open(filename, flags, function (err, fd) {
if (err) { return callback(err); }
fs.fsync(fd, function (errFS) {
fs.close(fd, function (errC) {
if (errFS || errC) {
var e = new Error('Failed to flush to storage');
e.errorOnFsync = errFS;
e.errorOnClose = errC;
return callback(e);
} else {
return callback(null);
}
});
});
});
}...
, tempFilename = filename + '~';
async.waterfall([
async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
, function (cb) {
storage.exists(filename, function (exists) {
if (exists) {
storage.flushToStorage(filename, function (err) { return cb(err); });
} else {
return cb();
}
});
}
, function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); });
...function mkdirP(p, opts, f, made) {
if (typeof opts === 'function') {
f = opts;
opts = {};
}
else if (!opts || typeof opts !== 'object') {
opts = { mode: opts };
}
var mode = opts.mode;
var xfs = opts.fs || fs;
if (mode === undefined) {
mode = _0777 & (~process.umask());
}
if (!made) made = null;
var cb = f || function () {};
p = path.resolve(p);
xfs.mkdir(p, mode, function (er) {
if (!er) {
made = made || p;
return cb(null, made);
}
switch (er.code) {
case 'ENOENT':
mkdirP(path.dirname(p), opts, function (er, made) {
if (er) cb(er, made);
else mkdirP(p, opts, cb, made);
});
break;
// In the case of any other error, just see if there's a dir
// there already. If so, then hooray! If not, then something
// is borked.
default:
xfs.stat(p, function (er2, stat) {
// if the stat fails, then that's super weird.
// let the original error be the failure reason.
if (er2 || !stat.isDirectory()) cb(er, made)
else cb(null, made);
});
break;
}
});
}...
* Check if a directory exists and create it on the fly if it is not the case
* cb is optional, signature: err
*/
Persistence.ensureDirectoryExists = function (dir, cb) {
var callback = cb || function () {}
;
storage.mkdirp(dir, function (err) { return callback(err); });
};
/**
* Return the path the datafile if the given filename is relative to the directory where Node Webkit stores
...readFile = function (path, options, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]);
if (!options || typeof options === 'function') {
options = { encoding: null, flag: 'r' };
} else if (typeof options === 'string') {
options = { encoding: options, flag: 'r' };
} else if (typeof options !== 'object') {
throwOptionsError(options);
}
var encoding = options.encoding;
assertEncoding(encoding);
var flag = options.flag || 'r';
if (!nullCheck(path, callback))
return;
var context = new ReadFileContext(callback, encoding);
context.isUserFd = isFd(path); // file descriptor ownership
var req = new FSReqWrap();
req.context = context;
req.oncomplete = readFileAfterOpen;
if (context.isUserFd) {
process.nextTick(function() {
req.oncomplete(null, path);
});
return;
}
binding.open(pathModule._makeLong(path),
stringToFlags(flag),
0o666,
req);
}...
// In-memory only datastore
if (self.inMemoryOnly) { return callback(null); }
async.waterfall([
function (cb) {
Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
storage.ensureDatafileIntegrity(self.filename, function (err) {
storage.readFile(self.filename, 'utf8', function (err, rawData) {
if (err) { return cb(err); }
try {
var treatedData = self.treatRawData(rawData);
} catch (e) {
return cb(e);
}
...rename = function (oldPath, newPath, callback) {
callback = makeCallback(callback);
if (!nullCheck(oldPath, callback)) return;
if (!nullCheck(newPath, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
binding.rename(pathModule._makeLong(oldPath),
pathModule._makeLong(newPath),
req);
}...
});
}
, function (cb) {
storage.writeFile(tempFilename, data, function (err) { return cb(err); });
}
, async.apply(storage.flushToStorage, tempFilename)
, function (cb) {
storage.rename(tempFilename, filename, function (err) { return cb(err); });
}
, async.apply(storage.flushToStorage, { filename: path.dirname(filename), isDir: true })
], function (err) { return callback(err); })
};
/**
...unlink = function (path, callback) {
callback = makeCallback(callback);
if (!nullCheck(path, callback)) return;
var req = new FSReqWrap();
req.oncomplete = callback;
binding.unlink(pathModule._makeLong(path), req);
}...
/**
* Ensure the workspace exists and the db datafile is empty
*/
module.exports.prepareDb = function (filename, cb) {
Persistence.ensureDirectoryExists(path.dirname(filename), function () {
fs.exists(filename, function (exists) {
if (exists) {
fs.unlink(filename, cb);
} else { return cb(); }
});
});
};
/**
...writeFile = function (path, data, options, callback_) {
var callback = maybeCallback(arguments[arguments.length - 1]);
if (!options || typeof options === 'function') {
options = { encoding: 'utf8', mode: 0o666, flag: 'w' };
} else if (typeof options === 'string') {
options = { encoding: options, mode: 0o666, flag: 'w' };
} else if (typeof options !== 'object') {
throwOptionsError(options);
}
assertEncoding(options.encoding);
var flag = options.flag || 'w';
if (isFd(path)) {
writeFd(path, true);
return;
}
fs.open(path, flag, options.mode, function(openErr, fd) {
if (openErr) {
callback(openErr);
} else {
writeFd(fd, false);
}
});
function writeFd(fd, isUserFd) {
var buffer = (data instanceof Buffer) ?
data : Buffer.from('' + data, options.encoding || 'utf8');
var position = /a/.test(flag) ? null : 0;
writeAll(fd, isUserFd, buffer, 0, buffer.length, position, callback);
}
}...
var b = browserify()
, srcPath = path.join(__dirname, 'src/lib/datastore.js');
b.add(srcPath);
b.bundle({ standalone: 'Nedb' }, function (err, out) {
if (err) { return cb(err); }
fs.writeFile(path.join(__dirname, 'out/nedb.js'), out, 'utf8',
function (err) {
if (err) {
return cb(err);
} else {
return cb(null, out);
}
});
});
...