Sometimes there is need to split content sent for log analytics API in certain size chunks to avoid hitting throttling limits
The logic is typically to avoid:
- Hitting throttling limit by aggressively sending single request for each single object
{"item":"1","content":"logdata"} - Hitting throttling limit by pure size of data (multiple objects in array)
{"item":"1","content":"logdata",{"...":"...."},,{"...":"...."},,{"...":"...."},
In my experience its rare to hit these throttling limits, especially nr 2. That being said, I like to use small code to split the data in chunks, which ensures (with right chunk size) I don’t need to retry the calls due to throttling (you can easily build retry logic too…)
- Due to Node.js asynchronous operation it can easily saturate throttling limits; By default items looped in array do not await for callback from function called on each item before proceeding to next (unless you want so, thus ’await’) – This might result easily in creating hundreds of HTTP request callbacks in the queue.
Solution (snippets)
Code below splits data in chunks, and creates remainder chunk for data that is not even size with the splitting value
splitArrayLogAnalytics/test.js at main · jsa2/splitArrayLogAnalytics (github.com)
var fs = require('fs')
var c = fs.readFileSync('testdata.json').toString()
var data = JSON.parse(c)
// Set count of throttle
Throttld(22)
var count = 0
async function Throttld(burstCount) {
var fullar = []
var burstArray = []
var residue = data.length % burstCount
console.log('residue is', residue)
for await (item of data) {
count++
burstArray.push(item)
if (count % burstCount == 0) {
console.log('chunk sent')
fullar.push(burstArray)
var burstArray = []
}
}
var resid = data.splice((data.length - residue), data.length)
fullar.push(resid)
console.log(fullar)
}
Git Repo with Log Analytics code
You can download full code examples here, but if you want to have snippets you can review samples below (Note sample code below does not include dependencies, no package.json file)
jsa2/splitArrayLogAnalytics (github.com)
Log Analytics example
splitArrayLogAnalytics/prod.js at main · jsa2/splitArrayLogAnalytics (github.com)
var {PushToAzureLogs} = require('./src/laClient')
var fs = require('fs')
const { argv } = require('process')
// read file so it parses
var f = fs.readFileSync('azuredata.json', 'utf16le').trim()
// Parse the file
var data = JSON.parse(f)
// If you use commandLineArgs then log name is taken from third argument
var LogType = argv[3]?.split('.json')[0] || 'monster'
//Get LA Config (Do not store stuff like this in prod)
var la = require('./config/key.json')
const laws = {
id:la.id,
key:la.key,
rfc1123date:(new Date).toUTCString(),
LogType
}
Throttld(600)
var count = 0
async function Throttld(burstCount) {
var burstArray = []
var residue = data.length % burstCount
console.log('residue is', residue)
// Everytime Modulo sum is 0, create burstArray
for await (item of data) {
count++
burstArray.push(item)
if (count % burstCount == 0) {
console.log('bursting')
await PushToAzureLogs(burstArray, laws).catch((error) => console.log(error))
var burstArray = []
}
}
// Handle Residuals in single call
var resid = data.splice((data.length - residue), data.length)
resid
if (resid.length > 0) {
await PushToAzureLogs(resid, laws).catch((error) => console.log(error))
}
}
Log Analytics Client code
//https://nodejs.org/api/crypto.html
//https://docs.microsoft.com/en-us/azure/azure-monitor/platform/data-collector-api
//https://stackoverflow.com/questions/44532530/encoding-encrypting-the-azure-log-analytics-authorization-header-in-node-js
const { default: axios } = require('axios')
const crypto = require('crypto')
function PushToAzureLogs (content,options) {
/* console.log(id) */
var {id,key,rfc1123date,LogType} = options
return new Promise ((resolve,reject) =>{
try {
//Checking if the data can be parsed as JSON
if ( JSON.parse(JSON.stringify(content)) ) {
var length = Buffer.byteLength(JSON.stringify(content),'utf8')
var binaryKey = Buffer.from(key,'base64')
var stringToSign = 'POST\n' + length + '\napplication/json\nx-ms-date:' + rfc1123date + '\n/api/logs';
//console.log(stringToSign)
var hash = crypto.createHmac('sha256',binaryKey)
.update(stringToSign,'utf8')
.digest('base64')
var authorization = "SharedKey "+id +":"+hash
/* require('fs').appendFileSync('key.txt', (hash + "\n")) */
var options= {
method:"post",
url:"https://"+ id + ".ods.opinsights.azure.com/api/logs?api-version=2016-04-01",
json:true,
headers:{
"content-type": "application/json",
"authorization":authorization,
"Log-Type":LogType,
"x-ms-date":rfc1123date,
"time-generated-field":"DateValue"
},
data:content
}
axios(options).then((data) => {
return resolve(data?.statusText || data?.statusCode)
}).catch((error) => {
return reject(error?.statusText || error?.statusCode)
})
}
//Catch error if data cant be parsed as JSON
} catch (err) {
return reject('no data sent to LA')
}
})
}
module.exports={PushToAzureLogs}
End
Hope somebody finds this handy, and just ping me if you find any errors in the code etc!
SecureCloudBlog 
Great article, where can i find the key needed in authorization header?
TykkääTykkää
Sorry for delay. I think The key can be found under at least two places in log analytics, one is the agent configuration
TykkääTykkää