Elasticsearch show all results using scroll in node js
You need to repeatedly call client.scroll
until no more records are returned. There's a good example in the elasticsearch documentation. I've reproduced their example code below, slightly modified to match your question
var allRecords = [];// first we do a search, and specify a scroll timeoutclient.search({ index: 'test', type: 'records', scroll: '10s', body: { query: { "match_all": {} } }}, function getMoreUntilDone(error, response) { // collect all the records response.hits.hits.forEach(function (hit) { allRecords.push(hit); }); if (response.hits.total !== allRecords.length) { // now we can call scroll over and over client.scroll({ scrollId: response._scroll_id, scroll: '10s' }, getMoreUntilDone); } else { console.log('all done', allRecords); }});
Thanks @Ceilingfish. Here's a modified ES6 version of the above using await
let allRecords = [];// first we do a search, and specify a scroll timeoutvar { _scroll_id, hits } = await esclient.search({ index: 'test', type: 'records', scroll: '10s', body: { query: { "match_all": {} }, _source: false }})while(hits && hits.hits.length) { // Append all new hits allRecords.push(...hits.hits) console.log(`${allRecords.length} of ${hits.total}`) var { _scroll_id, hits } = await esclient.scroll({ scrollId: _scroll_id, scroll: '10s' })}console.log(`Complete: ${allRecords.length} records retrieved`)
Query for getting all data from elastic search using Node.js client using scroll with async/await.
const elasticsearch = require('@elastic/elasticsearch');async function esconnection(){ let es = await new elasticsearch.Client({ node: "http://192.168.1.1:7200" }); return es;}async function getAllUserList(){ try{ let userArray = []; let query ={ "query":{ "match_all": {} } } let es = await esconnection(); let {body}= await es.search({ index: 'esIndex', type :"esIndexType", scroll :'2m', //# Specify how long a consistent view of the index should be maintained for scrolled search size: 100, // # Number of hits to return (default: 10) body: query }); let sid = body['_scroll_id'] let scroll_size = body['hits']['total'] let dataLength = body['hits']['hits'].length while (scroll_size > 0){ for(let i=0; i<dataLength;i++){ if(body['hits']['hits'][i]) { let userData = (body['hits']['hits'][i]['_source']) userArray.push(userData) } } sid = body['_scroll_id'] body = await es.scroll({ scrollId: sid, scroll: '10s' }) body=body.body scroll_size = (body['hits']['hits']).length; } es.close(); return userArray; } catch(error){ console.log("Code not working properly: ",`${error}`) }}