Script To Generate Test Data

From Observer GigaFlow Support | VIAVI Solutions Inc.
Jump to: navigation, search
//{"runPeriod":2000,"paused":false,
// "description":"Script to create flows"
// }
//Created by Kevin Wilkie 03/05/2022
var deviceCount=1;//up to 255
var flowCount=100;
var apps=2000;
var interfaces=10;
var debug = false;
var utils = Java.type('ros.CROSUtils');
var CFlowRecord = Java.type('ros.servers.CFlowRecord');
var deviceManager = actions.getDeviceManager();
var meta=utils.getJSONFromString("{}");
function createFlow(deviceip, interfaces, flows,apps) {
   var device = deviceManager.getDevice(utils.inet_atoi(deviceip), 5, new Date().getTime());
   for (var i = 0; i < flows; i++) {
       var flowRecord = new CFlowRecord();
       flowRecord.device = device;
       flowRecord.starttime = Date.now();
       flowRecord.now =  flowRecord.starttime;
       flowRecord.duration = getRandomInt(60, 0);
       flowRecord.srcport = getRandomInt(apps, 22);
       flowRecord.dstport =  flowRecord.srcport+1024;
       flowRecord.srcadd = utils.inet_atob("172.1."+(device.uniquedeviceid%2)+"."+getRandomInt(2,1));
       flowRecord.dstadd = utils.inet_atob("172.2.1."+(device.uniquedeviceid%10));
       flowRecord.outif = Math.abs(flowRecord.srcadd.intValue()%interfaces)+1;
       flowRecord.inif = Math.abs(flowRecord.dstadd.intValue()%interfaces)+1;
       flowRecord.proto = 6;
       flowRecord.packets = getRandomInt(10000, 1);
       flowRecord.bytes = parseInt( getRandomInt(100,1)*flowRecord.packets);
       // flowRecord.url="";
       // flowRecord.fwextcode="";
       // flowRecord.fwevent = actions.addFWEvent(read[headingHash.get("action")]);
       flowRecord.meta = meta;
       // log.warn(device.getIP() );
       // log.warn(device.getIP() + " " + flowRecord);
       // log.warn(device.getIP() + " " + flowRecord.device);
       device.process(flowRecord);
   }
}
function getRandomInt(m, offset) {
    return Math.floor(Math.random() * m) + offset;
}
for (var i=1;i<=deviceCount;i++) {
    createFlow("1.2.0."+i, interfaces, flowCount,apps);
}
log.warn("Creating test data of "+(deviceCount*flowCount)+" flows across "+deviceCount+" devices.");


//{"runPeriod":2000,"paused":false,
// "description":"Script to create flows"
// }
//Created by Kevin Wilkie 03/05/2022
//server,proto,port,device[],clients[]
var servers=[
 
 ["136.147.111.227",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.162.104.22","10.98.240.19"]],
   ["157.234.38.2",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","136.147.96.157","136.147.111.227"]],
 ["10.106.5.78",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.238.122.202"]],
 ["72.21.81.240",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.238.122.202"]],
 ["23.194.116.224",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.238.123.50","10.238.122.202"]],
 ["10.10.5.77",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","157.234.238.2"]],
 ["10.200.10.5",6,443,["2.0.0.1","2.0.0.2","2.0.0.3"],["10.230.248.241","10.230.248.242","10.230.248.243","10.230.248.244","172.21.40.15","10.3.91.233","10.200.1.29","10.52.16.11"]],
]
var deviceCount=10;//up to 255   1 device
var flowCount=1; // 1 flow every 10 seconds / device  100000
var apps=1;
var interfaces=1;
var debug = false;
var utils = Java.type('ros.CROSUtils');
var CFlowRecord = Java.type('ros.servers.CFlowRecord');
var deviceManager = actions.getDeviceManager();
var meta=utils.getJSONFromString("{}");
function createFlow(deviceip, interfaces, flows,proto,app,client,server) {
 //log.warn("Creating test data of "+deviceip);
  var device = deviceManager.getDevice(utils.inet_atoi(deviceip), 5, new Date().getTime());
  for (var i = 0; i < flows; i++) {
      var flowRecord = new CFlowRecord();
      flowRecord.device = device;
      flowRecord.starttime = Date.now();
      flowRecord.now =  flowRecord.starttime;
      flowRecord.duration = getRandomInt(50, 10);
      flowRecord.srcport = getRandomInt(apps, 65400);
      flowRecord.dstport =  app;
      flowRecord.srcadd = utils.inet_atob(client);
      flowRecord.dstadd = utils.inet_atob(server);
      flowRecord.outif = Math.abs(flowRecord.srcadd.intValue()%interfaces)+1;
      flowRecord.inif = Math.abs(flowRecord.dstadd.intValue()%interfaces)+1;
      flowRecord.proto = proto;
      flowRecord.packets = getRandomInt(100, 1);
      flowRecord.bytes = parseInt( getRandomInt(10,1)*flowRecord.packets);
      // flowRecord.url="";
      // flowRecord.fwextcode="";
      // flowRecord.fwevent = actions.addFWEvent(read[headingHash.get("action")]);
      flowRecord.meta = meta;
      // log.warn(device.getIP() );
      // log.warn(device.getIP() + " " + flowRecord);
      // log.warn(device.getIP() + " " + flowRecord.device);
    var clone = flowRecord.getClone();
    clone.packets=clone.packets/2;
    clone.bytes=clone.bytes/2;
    clone.nexthop=clone.srcadd
    clone.srcadd=clone.dstadd;
    clone.dstadd=clone.nexthop;
     var tempin= clone.srcport
     clone.srcport = clone.dstport
      clone.dstport =  tempin;
     tempin =clone.inif;
    clone.inif =clone.outif;
    clone.outif=tempin
    try{
      device.process(flowRecord);
       device.process(clone);
    }catch(err){log.warn(err);}
   }
}
function getRandomInt(m, offset) {
   return Math.floor(Math.random() * m) + offset;
}
for (var server=0;server<servers.length;server++) {
 log.warn("Creating test data of server"+server +" of "+servers.length);
 	for (var devices=0;devices<servers[server][3].length;devices++) {
     //log.warn("Creating test data of devices"+devices +" of "+servers[server][3].length);
 		for (var clients=0;clients<servers[server][4].length;clients++) {
         //log.warn("Creating test data of clients"+clients +" of "+servers[server][4].length);
   		createFlow(servers[server][3][devices], interfaces, flowCount,servers[server][1],servers[server][2],servers[server][4][clients],servers[server][0]);
     	}
 	}
}
log.warn("Creating test data of "+(deviceCount*flowCount)+" flows across "+deviceCount+" devices.");


Script to generate VPC flow data

var firstline="account-id action az-id bytes dstaddr dstport end flow-direction instance-id interface-id log-status packets pkt-dst-aws-service pkt-dstaddr pkt-src-aws-service pkt-srcaddr protocol region srcaddr srcport start sublocation-id sublocation-type subnet-id tcp-flags traffic-path type version vpc-id"
//var AwsParserV1 = actions.ros.awsmanager.getFakeParser("parser1",firstline); 
//log.warn(AwsParserV1);
var AwsParserV1 = Java.type('ros.cloud.aws.service.AwsParserV1');
var AwsS3Context= Java.type('ros.cloud.aws.bean.AwsS3Context');
var connection = actions.ros.awsmanager.connections.get("AKIAVXISGYQ3PKLFUKN3")
log.warn(connection);
var ctx= new AwsS3Context(connection);
ctx.cDeviceManager = actions.ros.deviceManager
ctx.customerData=actions.ros.getCustomer(0)
log.warn(ctx);
var parser =new AwsParserV1(null,ctx,firstline,null); 
var data=[
"931170224434 REJECT euw3-az3 1378 172.31.45.212 443 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 185.200.118.46 17 eu-west-3 185.200.118.46 48572 1678282173 - - subnet-0c776085afb6d6165 0 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 2121 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 64.62.197.62 6 eu-west-3 64.62.197.62 56395 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 60 172.31.45.212 6379 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 8.130.26.245 6 eu-west-3 8.130.26.245 46686 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 3593 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 89.248.165.193 6 eu-west-3 89.248.165.193 40080 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 11447 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 5.188.87.6 6 eu-west-3 5.188.87.6 56590 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 8185 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 176.111.174.97 6 eu-west-3 176.111.174.97 51956 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 44 172.31.45.212 3093 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 162.142.125.233 6 eu-west-3 162.142.125.233 3847 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 ACCEPT euw3-az3 40 147.135.11.167 55127 1678282232 egress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 147.135.11.167 - 172.31.45.212 6 eu-west-3 172.31.45.212 3389 1678282173 - - subnet-0c776085afb6d6165 4 2 IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 20465 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 45.143.200.50 6 eu-west-3 45.143.200.50 43768 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 7790 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 176.111.174.85 6 eu-west-3 176.111.174.85 51839 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 40 172.31.45.212 8663 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 94.102.61.28 6 eu-west-3 94.102.61.28 43907 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 44 172.31.45.212 21237 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 162.142.125.176 6 eu-west-3 162.142.125.176 37439 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41",
"931170224434 REJECT euw3-az3 44 172.31.45.212 6605 1678282232 ingress i-0cd573d1b20b69388 eni-044779596ea0079c8 OK 1 - 172.31.45.212 - 162.142.125.241 6 eu-west-3 162.142.125.241 13769 1678282173 - - subnet-0c776085afb6d6165 2 - IPv4 5 vpc-0ff6bbb263e391f41"
 ]
for (var i=0;i<data.length;i++){
log.warn(parser.awsS3Parse(data[i].split(" "),Date.now()))
}


Generate VPC flows from file and alter

var instancePrefix= "0cd573d1b20b";
var interfacePrefix="adadada";
var instances=0;//set to 0 to disable messing with files
var interfaces=0;//set to 0 to disable messing with files
var InputStreamReader= Java.type('java.io.InputStreamReader');
var BufferedReader = Java.type('java.io.BufferedReader');
var FileReader = Java.type('java.io.FileReader');
var File = Java.type('java.io.File');
var utils = Java.type('ros.CROSUtils');
var now = new Date()
var AwsParserV1 = Java.type('ros.cloud.aws.service.AwsParserV1');
var AwsS3Context= Java.type('ros.cloud.aws.bean.AwsS3Context');
 var connection = actions.ros.awsmanager.connections.get("AKIAVXISGYQ3PKLFUKN3")
var ctx= new AwsS3Context(connection);
ctx.cDeviceManager = actions.ros.deviceManager
ctx.customerData=actions.ros.getCustomer(0)
log.warn("start")
readFolder("c:/temp/sentry/")
function readFolder(location){
var folder = new File(location);
var listOfFiles = folder.listFiles();
 	for (var i = 0; i < listOfFiles.length; i++) {
 		if (listOfFiles[i].isFile()) {
   		log.warn(listOfFiles[i].getName()+" "+	listOfFiles[i].getAbsolutePath());
       	readFile(listOfFiles[i].getAbsolutePath())
}
}
}
var headerpos={};
var instanceid=-1;
var interfaceid=-1;
var flowstart=-1;
var flowend=-1;
function parseHeader(firstline){
 	headerpos={}
  	instanceid=-1;
	interfaceid=-1;
  	flowstart=-1;
	flowend=-1;
   var d = firstline.split(" ")
for (var i=0;i<d.length;i++){
headerpos[d[i]]=i;
     	if (d[i]==="instance-id"){instanceid=i;}
     	else if (d[i]==="interface-id"){interfaceid=i;}
     	else if (d[i]==="start"){flowstart=i;}
     	else if (d[i]==="end"){flowend=i;}
}
 	log.warn(JSON.stringify(headerpos));
 	log.warn("instanceid:"+instanceid+" interfaceid:"+interfaceid);
}
function readFile(filename){
 var start=utils.now();
 log.warn("start readFile "+filename)
 var count=1;
try {
var areader = objReader = new BufferedReader(new FileReader(filename));
var line = areader.readLine();
     	if (line!=null){
         parseHeader(line);
         var parser =new AwsParserV1(null,ctx,line,null); 
while ((line = areader.readLine()) != null) {
             	count++;
//log.warn(line)
parseLine(parser,line);
}
}
     	areader.close()
} catch ( e) {
areader.close()
}
log.warn("end readFile took "+(utils.now()-start)+" for "+count+" lines in "+filename)
} 
function parseLine(parser,line){
   var linedata=line.split(" ");
  	var now=parseInt(Date.now()/1000);
  	//log.warn(linedata.join(" "));
   if (instanceid>-1&&instances>0){
     	//log.warn("instanceid:"+instanceid+" interfaceid:"+interfaceid+" instances:"+instances+" interfaces:"+interfaces);
for(var instance=0;instance<instances;instance++){
       	//log.warn("instanceid:"+instanceid+" interfaceid:"+interfaceid+" instances:"+instances+" interfaces:"+interfaces);
          	linedata[instanceid]="i-"+instancePrefix+(10000+instance);
         	if (interfaceid>-1&&interfaces>0){
         		for(var ainterface=0;ainterface<interfaces;ainterface++){
             		//log.warn("instanceid:"+instanceid+" interfaceid:"+interfaceid+" instance:"+instance+" ainterface:"+ainterface);
             		linedata[interfaceid]="eni-"+interfacePrefix+(10000+instance)+(10000+ainterface);
              		if (flowstart>-1){	linedata[flowstart]=(now-60+ainterface)}
if (flowend>-1){	linedata[flowend]=(now-10-ainterface)}              
             		//log.warn(linedata.join(" "));
           		parser.awsS3Parse(linedata,Date.now());
       		}
         	}else{
             		parser.awsS3Parse(linedata,Date.now());
}
};
   }else{
parser.awsS3Parse(linedata,Date.now());
   }
}