Validate and Deduplicate Addresses

View Code

# valid_name returns true if and only if the object has a non-empty "name" key.
def valid_name: has("name") and .name != null and .name != "";

# valid_address returns true if and only if the object has a non-empty "address"
# key.
def valid_address: has("address") and .address != null and .address != "";

# valid_zip returns true if and only if the object has a "zip" key that is a
# 5-digit number or a 5 digit- and 4-digit number concatenated by a hypen
# (ZIP+4 format).
#
# It's possible that some of the allowed zip codes are not truly valid, to
# check more strictly we'd need a database of valid zip codes to compare
# against.
def valid_zip: has("zip") and .zip != null and (.zip | test("^[0-9]{5}(-[0-9]{4})?$"));

# valid returns true if and only if the requirements for each individual field
# are met.
def valid: valid_name and valid_address and valid_zip;

# duplicates groups valid records into sub-arrays, where in each sub-array all
# elements have identical name, address, and zip fields. The array is then
# flattened to produce a final array of all duplicate records.
def duplicates: . |
  # Ignore invalid records.
  map(select(valid)) |
  # Group into arrays of duplicate records.
  group_by({ name: .name, address: .address, zip: .zip }) |
  # Select only arrays of >1 element (if there is just a single element, then
  # there are no duplicates).
  map(select(length > 1)) |
  flatten;

{
  invalid: map(select(valid | not)),
  duplicate: duplicates
}