How to transfer Postgres Data from Query to S3 Efficiently How to transfer Postgres Data from Query to S3 Efficiently database database

How to transfer Postgres Data from Query to S3 Efficiently


This should get you ~90% there. I haven't tested this exact implementation and there may be a typo or two, but I have similar code running in production right now.

const { Transform } = require('json2csv');const { Client, Query } = require('pg')const { S3 } = require('aws-sdk');const { Passthrough } = require('stream')const client = new Client()const s3 = new S3({ region: 'us-east-1' });const opts = { fields: ['field1', 'field2', 'field3'] };const transformOpts = { highWaterMark: 8192, encoding: 'utf-8', objectMode: true };const transform = new Transform(opts, transformOpts);const passthrough = new Passthrough();transform.pipe(passthrough)client.connect()const query = new Query('SELECT field1, field2, field3 FROM table')client.query(query)query.on('row', row => {  transform.push(row);  console.log('row!', row) // { field1: 1, field2: 2, field3: 3 }})query.on('end', () => {  transform.push(null)  console.log('query done')})query.on('error', err => {  transform.end();  console.error(err.stack)})s3.upload({ Body: passthrough, Key: 'somefile.csv', Bucket: 'some_bucket' }).send((err, data) => {  if (err) {    console.error({ err });    passthrough.destroy(err);  } else {    console.log(`File uploaded and available at ${data.Location}`);    passthrough.destroy();  }});