Azure Log Analytics

Node.js and Azure Log Analytics Collector API – Sending data in pre-defined chunk sizes

Sometimes there is need to split content sent for log analytics API in certain size chunks to avoid hitting throttling limits

https://docs.microsoft.com/en-us/azure/azure-monitor/logs/data-collector-api#data-limits

The logic is typically to avoid:

  1. Hitting throttling limit by aggressively sending single request for each single object {"item":"1","content":"logdata"}
  2. Hitting throttling limit by pure size of data (multiple objects in array) {"item":"1","content":"logdata",{"...":"...."},,{"...":"...."},,{"...":"...."},

In my experience its rare to hit these throttling limits, especially nr 2. That being said, I like to use small code to split the data in chunks, which ensures (with right chunk size) I don’t need to retry the calls due to throttling (you can easily build retry logic too…)

  • Due to Node.js asynchronous operation it can easily saturate throttling limits; By default items looped in array do not await for callback from function called on each item before proceeding to next (unless you want so, thus ’await’) – This might result easily in creating hundreds of HTTP request callbacks in the queue.

Solution (snippets)

Code below splits data in chunks, and creates remainder chunk for data that is not even size with the splitting value

splitArrayLogAnalytics/test.js at main · jsa2/splitArrayLogAnalytics (github.com)

var fs = require('fs')
var c = fs.readFileSync('testdata.json').toString()
var data = JSON.parse(c)

// Set count of throttle 
Throttld(22)
var count = 0

async function Throttld(burstCount) {
    var fullar = []
    var burstArray = []
    var residue = data.length % burstCount

    console.log('residue is', residue)

    for await (item of data) {

        count++

        burstArray.push(item)

        if (count % burstCount == 0) {
            console.log('chunk sent')
            fullar.push(burstArray)
            var burstArray = []
        }
    }
    var resid = data.splice((data.length - residue), data.length)
    fullar.push(resid)
    console.log(fullar)
}

Git Repo with Log Analytics code

You can download full code examples here, but if you want to have snippets you can review samples below (Note sample code below does not include dependencies, no package.json file)

jsa2/splitArrayLogAnalytics (github.com)

Log Analytics example

splitArrayLogAnalytics/prod.js at main · jsa2/splitArrayLogAnalytics (github.com)

var {PushToAzureLogs} = require('./src/laClient')
var fs = require('fs')
const { argv } = require('process')

// read file so it parses 
var f = fs.readFileSync('azuredata.json', 'utf16le').trim()
// Parse the file
var data = JSON.parse(f)
// If you use commandLineArgs then log name is taken from third argument
var LogType = argv[3]?.split('.json')[0] || 'monster'
//Get LA Config (Do not store stuff like this in prod)
var la = require('./config/key.json')

const laws = {
    id:la.id,
    key:la.key,
    rfc1123date:(new Date).toUTCString(),
    LogType
}

Throttld(600)
var count = 0

async function Throttld(burstCount) {

    var burstArray = []
    var residue = data.length % burstCount

    console.log('residue is', residue)
    // Everytime Modulo sum is 0, create burstArray 
    for await (item of data) {

        count++
     
        burstArray.push(item)

        if (count % burstCount == 0) {
            console.log('bursting')
            await PushToAzureLogs(burstArray, laws).catch((error) => console.log(error))
            var burstArray = []
        }   
    }

     // Handle Residuals in single call
    var resid = data.splice((data.length - residue), data.length)
    resid
    if (resid.length > 0) {
        await PushToAzureLogs(resid, laws).catch((error) => console.log(error))
    }
     
}

Log Analytics Client code

//https://nodejs.org/api/crypto.html
//https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api
//https://stackoverflow.com/questions/44532530/encoding-encrypting-the-azure-log-analytics-authorization-header-in-node-js

const { default: axios } = require('axios')
const crypto = require('crypto')


function PushToAzureLogs (content,options) {
  /*   console.log(id) */
  var {id,key,rfc1123date,LogType} = options
  return new Promise ((resolve,reject) =>{
  
    try {
        //Checking if the data can be parsed as JSON
        if ( JSON.parse(JSON.stringify(content)) ) {

            var length = Buffer.byteLength(JSON.stringify(content),'utf8')
            var binaryKey = Buffer.from(key,'base64')
            var stringToSign = 'POST\n' + length + '\napplication/json\nx-ms-date:' + rfc1123date + '\n/api/logs';
            //console.log(stringToSign)
    
            var hash = crypto.createHmac('sha256',binaryKey)
            .update(stringToSign,'utf8')
            .digest('base64')
            var authorization = "SharedKey "+id +":"+hash
           /*  require('fs').appendFileSync('key.txt', (hash + "\n")) */
            var options= {
                method:"post",
                url:"https://"+ id + ".ods.opinsights.azure.com/api/logs?api-version=2016-04-01",
            json:true,
            headers:{
            "content-type": "application/json", 
            "authorization":authorization,
            "Log-Type":LogType,
            "x-ms-date":rfc1123date,
            "time-generated-field":"DateValue"
            },
            data:content    
            }

       
            axios(options).then((data) => {
                return resolve(data?.statusText || data?.statusCode)
            }).catch((error) => {
                return reject(error?.statusText || error?.statusCode)
            })
               

        }
        //Catch error if data cant be parsed as JSON
    } catch (err) {

        return reject('no data sent to LA')
    }

  })

           
}


module.exports={PushToAzureLogs}





End

Hope somebody finds this handy, and just ping me if you find any errors in the code etc!

2 comments on “Node.js and Azure Log Analytics Collector API – Sending data in pre-defined chunk sizes

  1. Great article, where can i find the key needed in authorization header?

    Tykkää

Vastaa

Täytä tietosi alle tai klikkaa kuvaketta kirjautuaksesi sisään:

WordPress.com-logo

Olet kommentoimassa WordPress.com -tilin nimissä. Log Out /  Muuta )

Google photo

Olet kommentoimassa Google -tilin nimissä. Log Out /  Muuta )

Twitter-kuva

Olet kommentoimassa Twitter -tilin nimissä. Log Out /  Muuta )

Facebook-kuva

Olet kommentoimassa Facebook -tilin nimissä. Log Out /  Muuta )

Muodostetaan yhteyttä palveluun %s

%d bloggaajaa tykkää tästä: