Elasticsearch show all results using scroll in node js Elasticsearch show all results using scroll in node js elasticsearch elasticsearch

Elasticsearch show all results using scroll in node js


You need to repeatedly call client.scroll until no more records are returned. There's a good example in the elasticsearch documentation. I've reproduced their example code below, slightly modified to match your question

var allRecords = [];// first we do a search, and specify a scroll timeoutclient.search({  index: 'test',  type: 'records',  scroll: '10s',  body: {     query: {         "match_all": {}     }  }}, function getMoreUntilDone(error, response) {  // collect all the records  response.hits.hits.forEach(function (hit) {    allRecords.push(hit);  });  if (response.hits.total !== allRecords.length) {    // now we can call scroll over and over    client.scroll({      scrollId: response._scroll_id,      scroll: '10s'    }, getMoreUntilDone);  } else {    console.log('all done', allRecords);  }});


Thanks @Ceilingfish. Here's a modified ES6 version of the above using await

let allRecords = [];// first we do a search, and specify a scroll timeoutvar { _scroll_id, hits } = await esclient.search({    index: 'test',    type: 'records',    scroll: '10s',    body: {        query: {            "match_all": {}        },        _source: false    }})while(hits && hits.hits.length) {    // Append all new hits    allRecords.push(...hits.hits)    console.log(`${allRecords.length} of ${hits.total}`)    var { _scroll_id, hits } = await esclient.scroll({        scrollId: _scroll_id,        scroll: '10s'    })}console.log(`Complete: ${allRecords.length} records retrieved`)


Query for getting all data from elastic search using Node.js client using scroll with async/await.

const elasticsearch = require('@elastic/elasticsearch');async function esconnection(){  let es =  await new elasticsearch.Client({    node: "http://192.168.1.1:7200"  });  return es;}async function getAllUserList(){    try{        let userArray = [];        let query ={            "query":{                "match_all": {}            }        }           let es = await esconnection();        let {body}=  await es.search({                    index: 'esIndex',                    type :"esIndexType",                               scroll :'2m', //# Specify how long a consistent view of the index should be maintained for scrolled search                    size: 100,    //  # Number of hits to return (default: 10)                    body: query                    });        let sid = body['_scroll_id']        let scroll_size = body['hits']['total']        let dataLength = body['hits']['hits'].length        while (scroll_size > 0){        for(let i=0; i<dataLength;i++){            if(body['hits']['hits'][i])            {            let userData = (body['hits']['hits'][i]['_source'])            userArray.push(userData)            }        }        sid = body['_scroll_id']        body = await es.scroll({            scrollId: sid,            scroll: '10s'        })        body=body.body        scroll_size = (body['hits']['hits']).length;        }        es.close();        return userArray;    }  catch(error){        console.log("Code not working properly: ",`${error}`)    }}