/ #javascript #node 

Avoiding recursion pitfalls: MongoDB nested query clause addition and removal

A case-study on where recursion can be useful for enterprise Node.js applications and how to avoid its common pitfalls like RangeError: Maximum call stack size exceeded.

The full repository for this post is on GitHub: github.com/HugoDF/mongo-query-clause-modification

We’ll be implementing a solution to 2 real-world problems:

Add an $or query clause to a MongoDB query

See the final code at ./src/add-or-clause.js.

The parameters are query and orClause.

query is a MongoDB query which might or might not already contain an $or and/or $and clause.

orClause is an object containing and $or clause (it’s a fully-fledged MongoDB query in its own right) eg.

const orClause = {
  $or: [
    {createdAt: {$exists: false}},
    {createdAt: someDate}
  ]
};

There is initially just 1 thing to look out for: 1. the query does not contain an $or clause 2. the query contains an $or clause

When there’s no $or clause in the query

If there is no $or clause, we can simply spread our orClause query and the query parameter, ie.

const newQuery = {
  ...query,
  ...orClause
};

That is unless there’s and $and in there somewhere, in which case we want to add our orClause to the $and:

const newQuery = {
  ...query,
  $and: [...query.$and, orClause]
};

When there’s an $or clause in the query

If there is an $or clause, we can’t just overwrite it, we need to $and the two $or queries.

We should also keep existing $and clause contents which yields:

const newQuery = {
  ...queryWithoutOrRemoved,
  $and: [
    ...(query.$and || []),
    { $or: query.$or },
    orClause
  ]
};

Full solution

This is also available at ./src/add-or-clause.js.

function addOrClause(query, orClause) {
  const {$or, ...queryRest} = query;
  if ($or) {
    return {
      ...queryRest,
      $and: [...(queryRest.$and || []), {$or}, orClause]
    };
  }

  if (queryRest.$and) {
    return {
      ...queryRest,
      $and: [...queryRest.$and, orClause]
    };
  }

  return {
    ...query,
    ...orClause
  };
}

module.exports = addOrClause;

Corresponding Test Suite

We can observe how the different cases map pretty directly to test cases.

const addOrClause = require('./add-or-clause');

test('should add the passed or clause if no $or on the current query', () => {
  const orClause = {$or: [{myField: 'value'}, {myField: null}]};
  const query = {foo: 'bar'};
  expect(addOrClause(query, orClause)).toEqual({
    $or: [{myField: 'value'}, {myField: null}],
    foo: 'bar'
  });
});
describe('when the query already has an $or', () => {
  test('should add the passed or clause to and $and that also contains the current query', () => {
    const orClause = {$or: [{myField: 'value'}, {myField: null}]};
    const query = {$or: [{foo: 'bar'}, {foo: {$exists: false}}]};
    expect(addOrClause(query, orClause)).toEqual({
      $and: [
        {$or: [{foo: 'bar'}, {foo: {$exists: false}}]},
        {
          $or: [{myField: 'value'}, {myField: null}]
        }
      ]
    });
  });
  describe('when the query has an $and', () => {
    test('should keep the $and, add the $or and the current query', () => {
      const orClause = {$or: [{myField: 'value'}, {myField: null}]};
      const query = {
        $or: [{hello: 'world'}],
        $and: [{foo: 'bar'}, {bar: 'baz'}]
      };
      expect(addOrClause(query, orClause)).toEqual({
        $and: [
          {foo: 'bar'},
          {bar: 'baz'},
          {$or: [{hello: 'world'}]},
          {$or: [{myField: 'value'}, {myField: null}]}
        ]
      });
    });
  });
});
describe('when the query has an $and query', () => {
  test('should add the new or clause to the $and', () => {
    const orClause = {$or: [{myField: 'value'}, {myField: null}]};
    const query = {$and: [{foo: 'bar'}, {bar: 'baz'}]};
    expect(addOrClause(query, orClause)).toEqual({
      $and: [
        {foo: 'bar'},
        {bar: 'baz'},
        {$or: [{myField: 'value'}, {myField: null}]}
      ]
    });
  });
});

Remove references to a field in an MongoDB query (potentially) using $or and $and

See the full solution at ./src/remove-field-references.js.

In this case we’re creating a function that takes 2 parameters: query (MongoDB query as above) and fieldName (name of the field we want to remove references to).

Remove top-level fields

The simplest thing to do is remove references to the field at the top-level of the object.

We can create a simple omit function using destructuring and recursion

const omit = (obj, [field, ...nextFields]) => {
  const {[field]: ignore, ...rest} = obj;
  return nextFields.length > 0 ? omit(rest, nextFields) : rest;
};

And use it:

const newQuery = omit(query, [fieldName]);

Remove fields in any $or clause

To remove fields in an $or clause (which is a fully-fledged query) is as simple as taking the $or value (which is an array) and running a recursion of the function onto it.

This will remove fields at the top-level of the $or sub-queries and in nest $or fields’ sub-queries.

We want to make sure to remove empty $or sub-queries, since { $or: [ { }, {} ]} is an invalid query.

We default the query’s $or to an empty array and check length before spreading it back into the newQuery. This is because { $or: [] } is an invalid query.

We’re also careful to remove the top-level $or when spreading filteredTopLevel so that if the new $or is an empty array, the old $or is ommitted.

function removeFieldReferences (query, fieldName) {
  const filteredTopLevel = omit(query, [fieldName]);

  const newOr = (filteredTopLevel.$or || [])
    .map(q => removeFieldReferences(q, fieldName))
    .filter(q => Object.keys(q).length > 0);

  return {
    ...omit(filteredTopLevel, ['$or']),
    ...(newOr.length > 0 ? {$or: newOr} : {})
  };
}

Remove fields in any $and clause

The rationale for the $and solution is the same as for the $or solution.

We recurse and check that we’re not generating an invalid query by omitting empty arrays and objects:

function removeFieldReferences (query, fieldName) {
  const filteredTopLevel = omit(query, [fieldName]);

  const newAnd = (filteredTopLevel.$and || [])
    .map(q => removeFieldReferences(q, fieldName))
    .filter(q => Object.keys(q).length > 0);

  return {
    ...omit(filteredTopLevel, ['$and']),
    ...(newAnd.length > 0 ? {$and: newAnd} : {})
  };
}

Check that we’re not likely to bust the stack

The actual implementation has a maxDepth 3rd parameter defaulted to 5.

When maxDepth is equal to 0, we return the query without any treatment (arguably we should run the top-level filter).

On recursive calls to removeFieldReferences we pass (q, fieldName, maxDepth - 1) so that we’re not going any deeper than we need to by accident.

This avoids RangeError: Maximum call stack size exceeded.

Final Implementation

This is also available at ./src/remove-field-references.js.

const omit = (obj, [field, ...nextFields]) => {
  const {[field]: ignore, ...rest} = obj;
  return nextFields.length > 0 ? omit(rest, nextFields) : rest;
};

function removeFieldReferences(query, fieldName, maxDepth = 5) {
  if (maxDepth <= 0) {
    return query;
  }

  const filteredTopLevel = omit(query, [fieldName]);

  const newOr = (filteredTopLevel.$or || [])
    .map(q => removeFieldReferences(q, fieldName, maxDepth - 1))
    .filter(q => Object.keys(q).length > 0);

  const newAnd = (filteredTopLevel.$and || [])
    .map(q => removeFieldReferences(q, fieldName, maxDepth - 1))
    .filter(q => Object.keys(q).length > 0);

  return {
    ...omit(filteredTopLevel, ['$or', '$and']),
    ...(newOr.length > 0 ? {$or: newOr} : {}),
    ...(newAnd.length > 0 ? {$and: newAnd} : {})
  };
}

module.exports = removeFieldReferences;

Corresponding Test Suite

We can observe how the different cases map pretty directly to test cases.

const removeFieldReferences = require('./remove-field-references');

test('should remove top-level fields', () => {
  const query = {
    hello: 'value'
  };
  const fieldName = 'hello';
  expect(removeFieldReferences(query, fieldName).hello).toBeUndefined();
});
test('should return passed query when maxDepth is hit (avoids busting the stack by default)', () => {
  const query = {
    hello: 'value'
  };
  const fieldName = 'hello';
  expect(removeFieldReferences(query, fieldName, 0)).toEqual(query);
});
test('should remove references to the field in top-level $or queries', () => {
  const query = {
    $or: [
      {hello: 'value', otherField: 'not-related'},
      {hello: 'othervalue', otherField: 'even-less-related'}
    ]
  };
  const fieldName = 'hello';
  expect(removeFieldReferences(query, fieldName)).toEqual({
    $or: [{otherField: 'not-related'}, {otherField: 'even-less-related'}]
  });
});
test('should remove $or clauses where the query becomes empty on omission of a field', () => {
  const query = {
    $or: [{hello: 'value'}, {otherField: 'not-related'}]
  };
  const fieldName = 'hello';
  expect(removeFieldReferences(query, fieldName)).toEqual({
    $or: [{otherField: 'not-related'}]
  });
});
test('should remove references to field in top-level queries inside of $and', () => {
  const query = {
    $and: [
      {hello: 'value', otherField: 'value'},
      {hello: 'other-value', otherField: 'value'}
    ]
  };
  const fieldName = 'hello';
  expect(removeFieldReferences(query, fieldName)).toEqual({
    $and: [{otherField: 'value'}, {otherField: 'value'}]
  });
});
test('should remove $and clause if all queries end up filtered out', () => {
  const query = {
    foo: 'bar',
    $and: [{hello: 'value'}, {hello: 'other-value'}]
  };
  const fieldName = 'hello';
  expect(removeFieldReferences(query, fieldName)).toEqual({foo: 'bar'});
});
test('should remove references to field in nested $or inside of $and', () => {
  const query = {
    $and: [
      {
        $or: [{hello: 'value'}, {hello: null}]
      },
      {otherField: 'not-related'}
    ]
  };
  const fieldName = 'hello';
  expect(removeFieldReferences(query, fieldName)).toEqual({
    $and: [{otherField: 'not-related'}]
  });
});

unsplash-logoTine Ivanič

Author

Hugo Di Francesco

A developer, working out of London writing CSS, JavaScript and Python.

Subscribe for Enterprise Node.js and JavaScript Guides

Build your web platform with modern Node.js/JavaScript best-practices, tools and patterns