How to fetch all items from dynamo table using scan

Below is the quick example of iterating through all records in dynamoDB table using scan operation.

As per amazon documentation, The data returned from a Query or Scan operation is limited to 1 MB; this means that if you scan a table that has more than 1 MB of data, you’ll need to perform another Scan operation to continue to the next 1 MB of data in the table. If you query for specific attributes that match values that amount to more than 1 MB of data, you’ll need to perform another Query request for the next 1 MB of data. The second query request uses a starting point (ExclusiveStartKey) based on the key of the last returned value (LastEvaluatedKey) so you can progressively query or scan for new data in 1 MB increments. The LastEvaluatedKey is null when the entire Query or Scan result set is complete (i.e. the operation processed the “last page”).

EmployeeRepository.java
@Service
public class EmployeeRepositoryImpl implements EmployeeRepository {

private static final Logger LOGGER = LoggerFactory.getLogger(EmployeeRepositoryImpl.class.getName());

private AmazonDynamoDBClient dynamoDBClient;

@Override
public List<Employee> getAllEmployees() {

    List<Employee> lstEmployees = new ArrayList<Employee>();

    try {

        ScanRequest scanRequest = new ScanRequest().withTableName(Employee.TABLE_NAME);
        ScanResult result = null;

        LOGGER.info("Loading data for All employees");

        do {

            if (result != null) {
                scanRequest.setExclusiveStartKey(result.getLastEvaluatedKey());
            }
            result = dynamoDBClient.scan(scanRequest);
            List<Map<String, AttributeValue>> rows = result.getItems();

            // Iterate through All rows
            for (Map<String, AttributeValue> mapEmployeeRecord : rows) {
                Employee employee = parseEmployeeInfo(mapEmployeeRecord);
                lstEmployees.add(employee);
            }

        } while (result.getLastEvaluatedKey() != null);

        LOGGER.info("Total Employee Records in database=" + lstEmployees.size());

    } catch (AmazonClientException ex) {
        LOGGER.error("Problem Connecting to DynamoDb: " + ex.getMessage());
        throw new RuntimeException(ex.getMessage(), ex);
    }

    return lstEmployees;
}

/*
 * Converts Map of Attribute values to Employee Domain Object
 */
private Employee parseEmployeeInfo(Map<String, AttributeValue> mapEmployeeRecord) {

    Employee employee = new Employee();

    try {

        AttributeValue employeeIdAttrValue = mapEmployeeRecord.get(Employee.EMPLOYEE_ID);
        AttributeValue firstNameAttrValue = mapEmployeeRecord.get(Employee.FIRST_NAME);
        AttributeValue lastNameAttrValue = mapEmployeeRecord.get(Employee.LAST_NAME);
        AttributeValue joiningDateAttrValue = mapEmployeeRecord.get(Employee.JOINING_DATE);
        AttributeValue locationAttrValue = mapEmployeeRecord.get(Employee.LOCATION);

        // Being primary key, should always be non null
        employee.setEmployeeId(Long.valueOf(employeeIdAttrValue.getN()));

        if (firstNameAttrValue != null) {
            employee.setFirstName(mapEmployeeRecord.get(Employee.FIRST_NAME).getS());
        }

        if (lastNameAttrValue != null) {
            employee.setLastName(mapEmployeeRecord.get(Employee.LAST_NAME).getS());
        }

        if (joiningDateAttrValue != null) {
            employee.setJoiningDate(mapEmployeeRecord.get(Employee.JOINING_DATE).getS());
        }

        if (locationAttrValue != null) {
            employee.setLocation(mapEmployeeRecord.get(Employee.LOCATION).getS());
        }

    } catch (NumberFormatException ex) {
        System.out.println(ex.getMessage());
    }

    return employee;
}
}

Below is the Domain class.

Employee.java
package com.slabs.insight.domain;

import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBAttribute;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey;
import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBTable;

@DynamoDBTable(tableName = Employee.TABLE_NAME)
public class Employee {
	
	public static final String TABLE_NAME = "insightapp-employee";
	public static final String EMPLOYEE_ID = "employeeId";
	public static final String FIRST_NAME = "firstName";
	public static final String LAST_NAME = "lastName";
	public static final String JOINING_DATE = "joiningDate";
	public static final String LOCATION = "location";
	
	private Long employeeId;
	
	private String firstName;
	
	private String lastName;
	
	private String joiningDate;
	
	private String location;
	
	public Employee() {
		//
	}
	
	@DynamoDBHashKey(attributeName = Employee.EMPLOYEE_ID)
    public Long getEmployeeId() {
		return employeeId;
	}

	public void setEmployeeId(Long id) {
		this.employeeId = id;
	}
	
	@DynamoDBAttribute(attributeName = Employee.FIRST_NAME)
	public String getFirstName() {
		return firstName;
	}

	public void setFirstName(String firstName) {
		this.firstName = firstName;
	}

	@DynamoDBAttribute(attributeName = Employee.LAST_NAME)
	public String getLastName() {
		return lastName;
	}

	public void setLastName(String lastName) {
		this.lastName = lastName;
	}

	@DynamoDBAttribute(attributeName = Employee.JOINING_DATE)
	public String getJoiningDate() {
		return joiningDate;
	}

	public void setJoiningDate(String joiningDate) {
		this.joiningDate = joiningDate;
	}

	@DynamoDBAttribute(attributeName = Employee.LOCATION)
	public String getLocation() {
		return location;
	}

	public void setLocation(String location) {
		this.location = location;
	}
}

Reference


Version History


Date Description
2016-11-17    Initial Version