Script To Generate Test Data
From Observer GigaFlow Support | VIAVI Solutions Inc.
//{"runPeriod":2000,"paused":false,
// "description":"Script to create flows"
// }
//Created by Kevin Wilkie 03/05/2022
var deviceCount=1;//up to 255
var flowCount=100;
var apps=2000;
var interfaces=10;
var debug = false;
var utils = Java.type('ros.CROSUtils');
var CFlowRecord = Java.type('ros.servers.CFlowRecord');
var deviceManager = actions.getDeviceManager();
var meta=utils.getJSONFromString("{}");
function createFlow(deviceip, interfaces, flows,apps) {
var device = deviceManager.getDevice(utils.inet_atoi(deviceip), 5, new Date().getTime());
for (var i = 0; i < flows; i++) {
var flowRecord = new CFlowRecord();
flowRecord.device = device;
flowRecord.starttime = Date.now();
flowRecord.now = flowRecord.starttime;
flowRecord.duration = getRandomInt(60, 0);
flowRecord.srcport = getRandomInt(apps, 22);
flowRecord.dstport = flowRecord.srcport+1024;
flowRecord.srcadd = utils.inet_atob("172.1."+(device.uniquedeviceid%2)+"."+getRandomInt(2,1));
flowRecord.dstadd = utils.inet_atob("172.2.1."+(device.uniquedeviceid%10));
flowRecord.outif = Math.abs(flowRecord.srcadd.intValue()%interfaces)+1;
flowRecord.inif = Math.abs(flowRecord.dstadd.intValue()%interfaces)+1;
flowRecord.proto = 6;
flowRecord.packets = getRandomInt(10000, 1);
flowRecord.bytes = parseInt( getRandomInt(100,1)*flowRecord.packets);
// flowRecord.url="";
// flowRecord.fwextcode="";
// flowRecord.fwevent = actions.addFWEvent(read[headingHash.get("action")]);
flowRecord.meta = meta;
// log.warn(device.getIP() );
// log.warn(device.getIP() + " " + flowRecord);
// log.warn(device.getIP() + " " + flowRecord.device);
device.process(flowRecord);
}
}
function getRandomInt(m, offset) {
return Math.floor(Math.random() * m) + offset;
}
for (var i=1;i<=deviceCount;i++) {
createFlow("1.2.0."+i, interfaces, flowCount,apps);
}
log.warn("Creating test data of "+(deviceCount*flowCount)+" flows across "+deviceCount+" devices.");
//{"runPeriod":2000,"paused":false,
// "description":"Script to create flows"
// }
//Created by Kevin Wilkie 03/05/2022
//server,proto,port,device[],clients[]
var servers=[
["136.147.111.227",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.162.104.22","10.98.240.19"]],
["157.234.38.2",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","136.147.96.157","136.147.111.227"]],
["10.106.5.78",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.238.122.202"]],
["72.21.81.240",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.238.122.202"]],
["23.194.116.224",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.238.123.50","10.238.122.202"]],
["10.10.5.77",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","157.234.238.2"]],
["10.200.10.5",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.3.91.233","10.200.1.29","10.52.16.11"]],
]
var deviceCount=10;//up to 255 1 device
var flowCount=1; // 1 flow every 10 seconds / device 100000
var apps=1;
var interfaces=1;
var debug = false;
var utils = Java.type('ros.CROSUtils');
var CFlowRecord = Java.type('ros.servers.CFlowRecord');
var deviceManager = actions.getDeviceManager();
var meta=utils.getJSONFromString("{}");
function createFlow(deviceip, interfaces, flows,proto,app,client,server) {
//log.warn("Creating test data of "+deviceip);
var device = deviceManager.getDevice(utils.inet_atoi(deviceip), 5, new Date().getTime());
for (var i = 0; i < flows; i++) {
var flowRecord = new CFlowRecord();
flowRecord.device = device;
flowRecord.starttime = Date.now();
flowRecord.now = flowRecord.starttime;
flowRecord.duration = getRandomInt(50, 10);
flowRecord.srcport = getRandomInt(apps, 65400);
flowRecord.dstport = app;
flowRecord.srcadd = utils.inet_atob(client);
flowRecord.dstadd = utils.inet_atob(server);
flowRecord.outif = Math.abs(flowRecord.srcadd.intValue()%interfaces)+1;
flowRecord.inif = Math.abs(flowRecord.dstadd.intValue()%interfaces)+1;
flowRecord.proto = proto;
flowRecord.packets = getRandomInt(100, 1);
flowRecord.bytes = parseInt( getRandomInt(10,1)*flowRecord.packets);
// flowRecord.url="";
// flowRecord.fwextcode="";
// flowRecord.fwevent = actions.addFWEvent(read[headingHash.get("action")]);
flowRecord.meta = meta;
// log.warn(device.getIP() );
// log.warn(device.getIP() + " " + flowRecord);
// log.warn(device.getIP() + " " + flowRecord.device);
var clone = flowRecord.getClone();
clone.packets=clone.packets/2;
clone.bytes=clone.bytes/2;
clone.nexthop=clone.srcadd
clone.srcadd=clone.dstadd;
clone.dstadd=clone.nexthop;
var tempin= clone.srcport
clone.srcport = clone.dstport
clone.dstport = tempin;
tempin =clone.inif;
clone.inif =clone.outif;
clone.outif=tempin
try{
device.process(flowRecord);
device.process(clone);
}catch(err){log.warn(err);}
}
}
function getRandomInt(m, offset) {
return Math.floor(Math.random() * m) + offset;
}
for (var server=0;server<servers.length;server++) {
log.warn("Creating test data of server"+server +" of "+servers.length);
for (var devices=0;devices<servers[server][3].length;devices++) {
//log.warn("Creating test data of devices"+devices +" of "+servers[server][3].length);
for (var clients=0;clients<servers[server][4].length;clients++) {
//log.warn("Creating test data of clients"+clients +" of "+servers[server][4].length);
createFlow(servers[server][3][devices], interfaces, flowCount,servers[server][1],servers[server][2],servers[server][4][clients],servers[server][0]);
}
}
}
log.warn("Creating test data of "+(deviceCount*flowCount)+" flows across "+deviceCount+" devices.");
Script to generate VPC flow data
var firstline="account-id action az-id bytes dstaddr dstport end flow-direction instance-id interface-id log-status packets pkt-dst-aws-service pkt-dstaddr pkt-src-aws-service pkt-srcaddr protocol region srcaddr srcport start sublocation-id sublocation-type subnet-id tcp-flags traffic-path type version vpc-id"
//var AwsParserV1 = actions.ros.awsmanager.getFakeParser("parser1",firstline);
//log.warn(AwsParserV1);
var AwsParserV1 = Java.type('ros.cloud.aws.service.AwsParserV1');
var AwsS3Context= Java.type('ros.cloud.aws.bean.AwsS3Context');
var connection = actions.ros.awsmanager.connections.get("AKIAVXISGYQ3PKLFUKN3")
log.warn(connection);
var ctx= new AwsS3Context(connection);
ctx.cDeviceManager = actions.ros.deviceManager
ctx.customerData=actions.ros.getCustomer(0)
log.warn(ctx);
var parser =new AwsParserV1(null,ctx,firstline,null);
var data=[
"931170224434 REJECT euw3-az3 1378 172.31.45.212 443 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 185.200.118.46 17 eu-west-3 185.200.118.46 48572 1678282173 - - subnet-0c776085afb6d6165 0 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 2121 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 64.62.197.62 6 eu-west-3 64.62.197.62 56395 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 60 172.31.45.212 6379 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 8.130.26.245 6 eu-west-3 8.130.26.245 46686 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 3593 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 89.248.165.193 6 eu-west-3 89.248.165.193 40080 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 11447 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 5.188.87.6 6 eu-west-3 5.188.87.6 56590 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 8185 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 176.111.174.97 6 eu-west-3 176.111.174.97 51956 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 44 172.31.45.212 3093 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 162.142.125.233 6 eu-west-3 162.142.125.233 3847 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 ACCEPT euw3-az3 40 147.135.11.167 55127 1678282232 egress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 147.135.11.167 - 172.31.45.212 6 eu-west-3 172.31.45.212 3389 1678282173 - - subnet-0c776085afb6d6165 4 2 IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 20465 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 45.143.200.50 6 eu-west-3 45.143.200.50 43768 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 7790 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 176.111.174.85 6 eu-west-3 176.111.174.85 51839 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 8663 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 94.102.61.28 6 eu-west-3 94.102.61.28 43907 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 44 172.31.45.212 21237 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 162.142.125.176 6 eu-west-3 162.142.125.176 37439 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 44 172.31.45.212 6605 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 162.142.125.241 6 eu-west-3 162.142.125.241 13769 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41"
]
for (var i=0;i<data.length;i++){
log.warn(parser.awsS3Parse(data[i].split(" "),Date.now()))
}
Generate VPC flows from file and alter
var instancePrefix= "0cd573d1b20b";
var interfacePrefix="adadada";
var instances=0;//set to 0 to disable messing with files
var interfaces=0;//set to 0 to disable messing with files
var InputStreamReader= Java.type('java.io.InputStreamReader');
var BufferedReader = Java.type('java.io.BufferedReader');
var FileReader = Java.type('java.io.FileReader');
var File = Java.type('java.io.File');
var utils = Java.type('ros.CROSUtils');
var now = new Date()
var AwsParserV1 = Java.type('ros.cloud.aws.service.AwsParserV1');
var AwsS3Context= Java.type('ros.cloud.aws.bean.AwsS3Context');
var connection = actions.ros.awsmanager.connections.get("AKIAVXISGYQ3PKLFUKN3")
var ctx= new AwsS3Context(connection);
ctx.cDeviceManager = actions.ros.deviceManager
ctx.customerData=actions.ros.getCustomer(0)
log.warn("start")
readFolder("c:/temp/sentry/")
function readFolder(location){
var folder = new File(location);
var listOfFiles = folder.listFiles();
for (var i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile()) {
log.warn(listOfFiles[i].getName()+" "+ listOfFiles[i].getAbsolutePath());
readFile(listOfFiles[i].getAbsolutePath())
}
}
}
var headerpos={};
var instanceid=-1;
var interfaceid=-1;
var flowstart=-1;
var flowend=-1;
function parseHeader(firstline){
headerpos={}
instanceid=-1;
interfaceid=-1;
flowstart=-1;
flowend=-1;
var d = firstline.split(" ")
for (var i=0;i<d.length;i++){
headerpos[d[i]]=i;
if (d[i]==="instance-id"){instanceid=i;}
else if (d[i]==="interface-id"){interfaceid=i;}
else if (d[i]==="start"){flowstart=i;}
else if (d[i]==="end"){flowend=i;}
}
log.warn(JSON.stringify(headerpos));
log.warn("instanceid:"+instanceid+" interfaceid:"+interfaceid);
}
function readFile(filename){
var start=utils.now();
log.warn("start readFile "+filename)
var count=1;
try {
var areader = objReader = new BufferedReader(new FileReader(filename));
var line = areader.readLine();
if (line!=null){
parseHeader(line);
var parser =new AwsParserV1(null,ctx,line,null);
while ((line = areader.readLine()) != null) {
count++;
//log.warn(line)
parseLine(parser,line);
}
}
areader.close()
} catch ( e) {
areader.close()
}
log.warn("end readFile took "+(utils.now()-start)+" for "+count+" lines in "+filename)
}
function parseLine(parser,line){
var linedata=line.split(" ");
var now=parseInt(Date.now()/1000);
//log.warn(linedata.join(" "));
if (instanceid>-1&&instances>0){
//log.warn("instanceid:"+instanceid+" interfaceid:"+interfaceid+" instances:"+instances+" interfaces:"+interfaces);
for(var instance=0;instance<instances;instance++){
//log.warn("instanceid:"+instanceid+" interfaceid:"+interfaceid+" instances:"+instances+" interfaces:"+interfaces);
linedata[instanceid]="i-"+instancePrefix+(10000+instance);
if (interfaceid>-1&&interfaces>0){
for(var ainterface=0;ainterface<interfaces;ainterface++){
//log.warn("instanceid:"+instanceid+" interfaceid:"+interfaceid+" instance:"+instance+" ainterface:"+ainterface);
linedata[interfaceid]="eni-"+interfacePrefix+(10000+instance)+(10000+ainterface);
if (flowstart>-1){ linedata[flowstart]=(now-60+ainterface)}
if (flowend>-1){ linedata[flowend]=(now-10-ainterface)}
//log.warn(linedata.join(" "));
parser.awsS3Parse(linedata,Date.now());
}
}else{
parser.awsS3Parse(linedata,Date.now());
}
};
}else{
parser.awsS3Parse(linedata,Date.now());
}
}