Skip to content

O
OpenXG-AIEngine
Commit fe3aad70 authored by zcq98's avatar zcq98
Browse files
Options

flow type classification

parent 11a5ba5d
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1582 additions and 0 deletions
measurement/oaiflowclassification/.gitignore 0 → 100644
View file @ fe3aad70
__pycache__
oai_client/__pycache__
.py38
\ No newline at end of file
measurement/oaiflowclassification/classify.py 0 → 100644
View file @ fe3aad70
"""
预测函数：随机森林
每次运行前，检查：
四个需要修改的地方，命名是否正确
最后的运行模式是否正确
做预测
1. 直接使用之前的模型做预测 分类结果和五元组封装起来
2. 做一个线程池 多线程接收发送来的特征信息
"""
from common_utils import * # 修改了
from argparse import ArgumentParser
from collections import namedtuple
from typing import List, Dict
from datetime import datetime
from path_utils import get_prj_root
import numpy as np
# from sklearn.externals import joblib
import joblib
import pprint
import socket
import string
from sklearn.ensemble import RandomForestClassifier # 训练模型
random.seed(datetime.now())
model_dir = os.path.join(get_prj_root(), "classify/models") # 修改：模型model文件夹路径
predict_model_pkl = os.path.join(model_dir, "dt2.pkl") # 修改：模型的版本，只用修改此处就行
Instance = namedtuple("Instance", ["features", "label"]) # 实例
dirs = {
"video": "./tmp/dt/video",
"iot": "./tmp/dt/iot",
"voip": "./tmp/dt/voip",
"AR": "./tmp/dt/AR",
}
instances_dir = os.path.join(get_prj_root(), "classify/instances") # 修改：instances路径
# 通过元组测试
test_flow = (1, 2, 3, 4, 5, 6, 7, 8, 6, 10)
test_flow2 = [221.0, 1350.0, 640.0, 376.26798960315506, 543.0,
21257877.349853516, 4793407917.022705, 1263437211.5135193,
2039103758.0566826, 119541525.84075928]
def train_and_predict():
iot = load_pkl(os.path.join(instances_dir, "iot.pkl")) # 不同实例的pkl是不同特征的
videos = load_pkl(os.path.join(instances_dir, "video.pkl"))
voip = load_pkl(os.path.join(instances_dir, "voip.pkl"))
AR = load_pkl(os.path.join(instances_dir, "AR.pkl"))
for i in videos:
assert i.label == 0
for i in iot:
assert i.label == 1
for i in voip:
assert i.label == 2
for i in AR:
assert i.label == 3
# print(videos)
debug("# iot instances {}".format(len(iot)))
debug("# video instances {}".format(len(videos)))
debug("# VOIP instances {}".format(len(voip)))
debug("# AR instances {}".format(len(AR)))
random.shuffle(voip) # 打乱排序
random.shuffle(iot)
random.shuffle(videos)
random.shuffle(AR)
n_video_train = int(len(videos) * 0.7)
n_video_test = len(videos) - n_video_train
video_train = videos[:n_video_train]
video_test = videos[n_video_train:]
iot_train = iot[:n_video_train]
iot_test = iot[len(iot) - len(video_test):]
voip_train = voip[:n_video_train]
voip_test = voip[len(voip) - len(video_test):]
AR_train = AR[:n_video_train]
AR_test = AR[len(AR) - len(video_test):]
info("#video train {}".format(len(video_train)))
info("#iot train {}".format(len(iot_train)))
info("#voip train {}".format(len(voip_train)))
info("#AR train {}".format(len(AR_train)))
train = []
train.extend(iot_train)
train.extend(video_train)
train.extend(voip_train)
train.extend(AR_train)
random.shuffle(train)
train_x = [x.features for x in train]
train_y = [x.label for x in train]
# test 1:1
test = []
info("#video test {}".format(len(video_test)))
info("#iot test {}".format(len(iot_test)))
info("#voip test {}".format(len(voip_test)))
info("#AR test {}".format(len(AR_test)))
test.extend(video_test)
test.extend(iot_test)
test.extend(voip_test)
test.extend(AR_test)
random.shuffle(test)
test_x = [t.features for t in test]
test_y = [t.label for t in test]
# 训练以及预测
predict_model = RandomForestClassifier(oob_score=True) # 引入训练方法
predict_model.fit(train_x, train_y) # 队训练数据进行拟合
predicts = predict_model.predict(test_x)
"""
dt = DT()
dt.fit((train_x, train_y))
predicts = dt.predict(test_x)
"""
"""
# 打印预测的结果
print(predicts)
print("-------------------------------")
"""
# 保存模型
fn_name = os.path.join(model_dir, predict_model_pkl)
joblib.dump(predict_model, predict_model_pkl)
# 评价模型
count = 0
for idx in range(len(test_x)):
if int(predicts[idx]) == int(test_y[idx]):
count += 1
# print(count / len(test_x))
return count / len(test_x)
def classify_flows(mode: 'int', predict_dir):
"""
该函数用于训练模型并且测试模型的准确度 或者 预测结果
:param mode: 0--训练模型 1--预测和分类流并返回
:param predict_dir: 待预测的流的目录下的pkl文件
:return: 待分类的流的分类结果列表
"""
# 判断是只训练模型 还是 只是预测结果
if mode == 0:
# 此时训练使用数据训练模型 并且 保存模型 评价模型
times = 10
sum_predict = 0
for _ in range(times):
res = train_and_predict()
sum_predict = sum_predict + res
print("模型准确率为:", sum_predict / times)
else:
# 使用传递的文件来预测结果并且返回
predict = load_pkl(os.path.join(predict_dir, "predict2.pkl"))
test = []
info("#video test {}".format(len(predict)))
test.extend(predict)
# random.shuffle(test)
test_x = [t.features for t in test]
predict_model = joblib.load(predict_model_pkl)
predict_result = predict_model.predict(test_x)
res_list = identify_classification(predict_result)
return res_list
def classify_flow_list(flow_list):
"""
该方法用于分类为元组的流
格式：[[1, 2, 3, 4, 5, 6, 7, 8, 6, 10], ["五元组"]]
"""
test_x = [flow_list[0]]
predict_model = joblib.load(predict_model_pkl)
predict_result = predict_model.predict(test_x)
# 定义结果的变量 res = ["五元组", "分类结果"]
res = []
res.append(flow_list[1][0])
# 得到字符串的结果
if predict_result == 0:
res.append("videos")
elif predict_result == 1:
res.append("iot")
elif predict_result == 2:
res.append("voip")
else:
res.append("AR")
return res
def change_result_to_integer_list(result_list):
# 将这个格式 res = ["五元组", "分类结果"] 改为[srcIP, dstIP, srcPort, dstPort, protocol, flowType]
integer_list = []
string_list = result_list[0].split()
# 添加源ip 转换为int的
integer_list.append(ipToLong(string_list[0]))
# 添加目的ip 转换为int的
integer_list.append(ipToLong(string_list[1]))
# 添加源端口 转换为int的
integer_list.append(int(string_list[2]))
# 添加目的端口 转换为int的
integer_list.append(int(string_list[3]))
# 添加协议类型
#if string_list[4] == 'TCP':
# integer_list.append(1)
#elif string_list[4] == 'UDP':
# integer_list.append(2)
integer_list.append(int(string_list[4]))
# 添加分类结果
integer_list.append(result_list[1])
return integer_list
#将字符串形式的ip地址转成整数类型。
def ipToLong(ip_str):
#print map(int,ip_str.split('.'))
ip_long = 0
for index,value in enumerate(reversed([int(x) for x in ip_str.split('.')])):
ip_long += value<<(8*index)
return ip_long
def identify_classification(predict_result):
"""
该函数将分类结果的标签转换为具体内容字符串的结果
:param predict_result:标签分类结果
:return: 字符串分类结果
"""
res_list = []
for label in predict_result:
if label == 0:
res_list.append("videos")
elif label == 1:
res_list.append("iot")
elif label == 2:
res_list.append("voip")
elif label == 3:
res_list.append("AR")
return res_list
if __name__ == '__main__':
"""
测试 格式转换
# 训练模型
# classify_flows(mode=0, path=instances_dir)
# 预测结果
predict_dir = os.path.join(get_prj_root(), "classify/predict") # 修改：instances路径
predict_result_list = classify_flows(mode=1, predict_dir=predict_dir)
pprint.pprint(predict_result_list)
"""
list1 = ['54.52.52.53 51.49.50.44 8242 12596 UDP', 'iot']
res = change_result_to_integer_list(list1)
print(res)
measurement/oaiflowclassification/common_utils.py 0 → 100644
View file @ fe3aad70
import json
import os
import pickle
import random
from pathlib import Path
import loguru
import numpy as np
logger = loguru.logger
info = logger.info
debug = logger.debug
err = logger.error
def check_dir(dir_name):
if not Path(dir_name).is_dir():
raise FileNotFoundError
def check_file(fn):
if not Path(fn).is_file():
raise FileNotFoundError
def file_exsit(fn):
return Path(fn).is_file()
def dir_exsit(fn):
return Path(fn).is_dir()
def gaussion(mean: float, std_var: float, size=1):
if size == 1:
return np.random.normal(mean, std_var)
return np.random.normal(mean, std_var, size)
def exp(mean: float, size=1):
if 1 == size:
return np.random.normal(mean)
return np.random.normal(mean, size)
def uniform(low, up, size=1):
if 1 == size:
return np.random.uniform(low, up)
return np.random.uniform(low, up, size)
def load_pkl(filename):
if Path(filename).is_file():
data = None
with open(filename, 'rb') as file:
data = pickle.load(file)
file.close()
return data
raise FileNotFoundError
def save_pkl(filename, obj, overwrite=True):
def write():
with open(filename, 'wb') as file:
pickle.dump(obj, file)
file.flush()
file.close()
if Path(filename).is_file() and overwrite:
write()
return
write()
def load_json(filename):
if Path(filename).is_file():
with open(filename) as f:
return json.load(f)
raise FileNotFoundError
def save_json(filename, obj, overwrite=True):
def write():
with open(filename, 'w', encoding="utf8") as file:
json.dump(obj, file, indent=4)
if Path(filename).is_file and overwrite:
write()
return
write()
def is_digit(x: str) -> bool:
try:
float(x)
return True
except:
return False
def normalize(x):
mi = min(x)
ma = max(x)
diff = ma - mi
x = [(xx - mi) / diff for xx in x]
return x
if __name__ == "__main__":
pass
measurement/oaiflowclassification/database/measure.sql 0 → 100644
View file @ fe3aad70
/*
Navicat Premium Data Transfer
Source Server : enb2
Source Server Type : MySQL
Source Server Version : 50733
Source Host : 192.168.31.50:3306
Source Schema : mytestdb
Target Server Type : MySQL
Target Server Version : 50733
File Encoding : 65001
Date: 21/06/2021 20:11:19
*/
SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for measure
-- ----------------------------
DROP TABLE IF EXISTS `measure`;
CREATE TABLE `measure` (
`srcIP` bigint(20) NOT NULL,
`dstIP` bigint(20) NOT NULL,
`srcPort` smallint(6) UNSIGNED NOT NULL,
`dstPort` smallint(6) UNSIGNED NOT NULL,
`protocol` tinyint(4) UNSIGNED NOT NULL,
`averBytes` double(20, 1) NULL DEFAULT NULL,
`averPkts` double(20, 1) NULL DEFAULT NULL,
`flowType` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
PRIMARY KEY (`srcIP`, `dstIP`, `srcPort`, `dstPort`, `protocol`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = DYNAMIC;
-- ----------------------------
-- Records of measure
-- ----------------------------
INSERT INTO `measure` VALUES (167772418, 167772417, 40000, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40001, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40002, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40003, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40004, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40005, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40006, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40007, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40008, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40009, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40010, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40011, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40012, 11111, 6, 21470.0, 14.4, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40013, 11111, 6, 29806.0, 20.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40014, 11111, 6, 25293.2, 17.4, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40100, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40101, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40102, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40103, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40104, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40105, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40106, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40107, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40108, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40109, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40110, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40111, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40112, 11111, 6, 26057.2, 17.6, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40113, 11111, 6, 29806.0, 20.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40114, 11111, 6, 22774.0, 15.6, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40200, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40201, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40202, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40203, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40204, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40205, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40206, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40207, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40208, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40209, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40210, 11111, 6, 0.0, 0.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40211, 11111, 6, 5097.6, 3.4, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40212, 11111, 6, 29518.4, 20.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40213, 11111, 6, 29806.0, 20.0, NULL);
INSERT INTO `measure` VALUES (167772418, 167772417, 40214, 11111, 6, 24412.8, 16.8, NULL);
SET FOREIGN_KEY_CHECKS = 1;
measurement/oaiflowclassification/database/test.py 0 → 100644
View file @ fe3aad70
'''创建数据库'''
import pymysql
#打开数据库连接，不需要指定数据库，因为需要创建数据库
conn = pymysql.connect(host = 'localhost',user = "root",passwd = "123456")
#获取游标
cursor=conn.cursor()
#创建pythonBD数据库
cursor.execute('CREATE DATABASE IF NOT EXISTS pythonDB DEFAULT CHARSET utf8 COLLATE utf8_general_ci;')
cursor.close()#先关闭游标
conn.close()#再关闭数据库连接
print('创建pythonBD数据库成功')
measurement/oaiflowclassification/instructions.txt 0 → 100644
View file @ fe3aad70
由于路径的问题，在import上作了一些删除
在classify文件夹下是运行的数据
先运行parser产生解析PCAP之后的文件，放在./tmp/dt的目录下，statistics.json为各个类型流的数据
再运行train，先把最下面的模式设置为1，产生对statistics.json中包大小，包间隔进行处理产生特征，保存在instances文件夹的pkl中
最后是运行predict，预测
train_dt_1
设置的是8个特征的（前四个包大小，后四个包间隔）+决策树 。。。
特征与标签都保存到instances文件夹，
训练模型保存到models文件夹中
正确率可以达到90%
加了PCA降维之后为88.6%
train_dt_2
设置的是10个特征的（前五个包大小，后五个包间隔）：最小值，最大值，平均值，方差，中位数 +决策树
特征与标签都保存到instances2文件夹，
训练模型保存到models2文件夹中
正确率可以达到91.73%
加了PCA降维之后为88.7%
train_dt_3
11个特征（前五个包大小，后五个包间隔）：最小值，最大值，平均值，方差，中位数 ；每秒包的数量
预测函数：决策树
特征与标签都保存到instances3文件夹
训练模型保存到models3文件夹中
正确率可以达到91.7%
加了PCA降维之后为88.7%
instances装的是 8个特征的（前四个包大小，后四个包间隔）
pkl文件其实可以处理成Excel表格的形式，然后用MATLAB来跑
predit_1_1 K邻近+8特征 85%
predit_1_2 K邻近+10特征 86.59%
predit_2 支持向量机 + 8特征 25.9%
predict_3 随机森林 +8特征 93.5%
predict_3 随机森林 +10特征 94%
predict_3 随机森林 +11特征 93.5%
models文件下
model_predict文件夹装的是各种预测函数
dt1为 K邻近+8特征 85%
dt1_2 为 K邻近+10特征 86.59%
dt3_1 为 随机森林 +8特征 93.5%
dt3_2 为 随机森林 +10特征 94%
有后缀_9 说明是用9个窗口
measurement/oaiflowclassification/main.py 0 → 100644
View file @ fe3aad70
from classify import *
from multithread_server import TcpServer, boot_server
from pool import *
from save_result import *
import json
def load_flows():
# 一直执行
con = connect_database(DB_ADDR, USER_NAME, USER_PASSWORD, DB_NAME) # 建立数据库的连接
while True:
# 判断是否停止识别
"""
mutex.acquire()
if len(STOP_CLASSIFY) == 1:
print("程序结束")
mutex.release()
break
mutex.release()
"""
mutex.acquire()
if PREDECT_FLOWS:
# 读出所有的结果
for feature in PREDECT_FLOWS:
if type(feature) == list:
# print("type: ", type(feature))
# 预测结果
predict_result = classify_flow_list(feature)
# 将分类结果放到全局变量中
# if predict_result not in CLASSIFY_RESULT:
# CLASSIFY_RESULT.append(predict_result)
#print("缓冲池中有: ", len(CLASSIFY_RESULT), "个已经分类好的结果，请取走")
# 将分类结果转为整数
integer_list = change_result_to_integer_list(predict_result)
# 存入数据库
mysqldb_insert(con, integer_list[0], integer_list[1], integer_list[2], integer_list[3], integer_list[4], integer_list[5])
print(integer_list, "存入数据库成功！")
print("分类结果为：", integer_list)
# 清空列表
PREDECT_FLOWS.clear()
mutex.release()
# 定义一个函数作为主线程 让其他的守护他 判断关键字close后关闭主线称
def main_threading():
global STOP_CLASSIFY
# 创建线程
t_server = threading.Thread(target = boot_server)
t_classify = threading.Thread(target = load_flows)
# 设置其他的为守护线程
t_server.setDaemon(True)
t_classify.setDaemon(True)
# 启动线程
t_server.start()
t_classify.start()
while True:
mutex.acquire()
if len(STOP_CLASSIFY) == 1:
print("程序结束")
mutex.release()
break
mutex.release()
if __name__ == "__main__":
main_threading()
measurement/oaiflowclassification/model.py 0 → 100644
View file @ fe3aad70
from sklearn.tree import DecisionTreeClassifier
from path_utils import get_prj_root
from pathlib import Path
import os
from common_utils import save_pkl, load_pkl, info #修改了
import numpy as np
import random
root_dir = get_prj_root()
model_dir = os.path.join(root_dir, "models")
class Classifier:
# def __init__(self,fn_name=None):
# self.model=None
# if fn_name is not None:
# self.load_model(fn_name)
def fit(self, data):
raise NotImplementedError
def predict(self, features):
raise NotImplementedError
def save_model(self, fn_name):
raise NotImplementedError
def load_model(self, fn_name):
raise NotImplementedError
'''
min_pkt|max_pkt|mean_pkt|var_pkt
min_idt|max_idt|mean_idt|var_idt
'''
class DT(Classifier):
def __init__(self):
super(DT, self).__init__()
self.model: DecisionTreeClassifier = DecisionTreeClassifier()
def fit(self, data):
assert len(data) == 2
features = data[0]
y = data[1]
assert len(features) == len(y)
info("# instances {}".format(len(features)))
self.model.fit(features, y)
def predict(self, features):
# info("# instances {}".format(len(features)))
return self.model.predict(features)
def save_model(self, fn_name):
if self.model is None: return
fn_name = os.path.join(model_dir, fn_name)
save_pkl(fn_name, self.model)
def load_model(self, fn_name):
fn_name = os.path.join(model_dir, fn_name)
self.model: DecisionTreeClassifier = load_pkl(fn_name)
class Dumb(Classifier):
def fit(self, data):
pass
def predict(self, features):
if random.random() >= 0.5:
return 1
return 0
def save_model(self, fn_name):
pass
def load_model(self, fn_name):
pass
if __name__ == '__main__':
from sklearn.datasets import load_iris
x, y = load_iris(return_X_y=True)
model = DT()
model.fit((x, y))
model.save_model("test.pkl")
model.load_model("test.pkl")
measurement/oaiflowclassification/multithread_client.py 0 → 100644
View file @ fe3aad70
from socket import *
from threading import Thread
from random import choice
import json
import time
BUFFER_SIZE = 8192
# 随机发送数据测试的库:
data_list = [
[[221.0, 1350.0, 640.0, 376.26798960315506, 543.0, 21257877.349853516,
4793407917.022705, 1263437211.5135193, 2039103758.0566826, 119541525.84075928],["10.0.0.0 10.0.0.1 6666 7771 UDP"]],
[[171.0, 1460.0, 888.4, 514.1558518581695, 853.0, 11920.928955078125,
17442424058.914185, 4385495722.293854, 7538530505.708111, 49773454.666137695],["10.0.0.0 10.0.0.2 6666 7772 TCP"]],
[[498.0, 1460.0, 1088.8, 455.13356281425786, 1460.0, 36954.87976074219,
43512821.197509766, 11164724.826812744, 18679961.749486398, 554561.6149902344],["10.0.0.0 10.0.0.3 6666 7773 UDP"]],
[[498.0, 1460.0, 1088.8, 455.13356281425786, 1460.0, 30994.415283203125,
44764995.57495117, 16246497.631072998, 18385686.144529935, 10095000.267028809],["10.0.0.0 10.0.0.4 6666 7774 TCP"]],
[[566.0, 1460.0, 1281.2, 357.59999999999997, 1460.0, 22172.927856445312,
21413803.100585938, 5526959.8960876465, 9175244.715715846, 335931.77795410156],["10.0.0.0 10.0.0.5 6666 7775 UDP"]],
[[69.0, 629.0, 246.6, 205.1522361564699, 172.0, 190973.28186035156,
457492113.1134033, 219977498.0545044, 220039359.48775682, 211113452.91137695],["10.0.0.6 10.0.0.1 7776 6666 TCP"]],
[[69.0, 278.0, 137.8, 78.91108920804477, 85.0, 144958.49609375,
462785959.2437744, 210317969.3222046, 211827960.3342278, 189170479.7744751],["10.0.0.7 10.0.0.1 7777 6666 UDP"]],
[[302.0, 1460.0, 807.6, 537.1113850962387, 498.0, 12874.603271484375,
205285072.32666016, 56649982.92922974, 86251862.02980827, 10650992.393493652],["10.0.0.8 10.0.0.1 7778 6666 TCP"]],
[[266.0, 1460.0, 892.8, 490.89730086852177, 780.0, 24080.276489257812,
301223039.6270752, 76725006.10351562, 129634012.70485088, 2826452.2552490234],["10.0.0.9 10.0.0.1 7779 6666 UDP"]],
[[514.0, 1460.0, 1191.6, 371.89762032043177, 1460.0, 10013.580322265625,
890016.5557861328, 236511.23046875, 377434.79666208074, 23007.39288330078],["10.0.0.10 10.0.0.1 8000 6666 TCP"]],
"hhh", "exit", "send", "send"]
class TcpClient(object):
"""Tcp客户端"""
def __init__(self, IP="127.0.0.1", Port=5002):
"""初始化对象"""
self.code_mode = "utf-8" #收发数据编码/解码格式
self.IP = IP
self.Port = Port
self.my_socket = socket(AF_INET, SOCK_STREAM) #创建socket
def run(self):
"""启动"""
self.my_socket.connect((self.IP, self.Port)) #连接服务器
tr = Thread(target=self.recv_data) #创建线程收数据
ts = Thread(target=self.send_data) #创建线程发数据
tr.start() #开启线程
ts.start()
def recv_data(self):
"""收数据"""
while True:
data = self.my_socket.recv(BUFFER_SIZE).decode(self.code_mode)
if data:
if data == "close connecting":
print("已下线")
break
else:
print("{}".format(data))
else:
break
self.my_socket.close()
def send_data(self):
"""发数据"""
while True:
# 接收数据
data = choice(data_list)
if type(data) == list:
print("已发送五元组{}的特征".format(data[1][0]))
else:
print("发送命令: ", data)
# 两s发送一次 测试
time.sleep(2)
# 判断data的类型 如果是列表用 json传输 字符串就直接传输
if type(data) == list:
send_dat = json.dumps(data)
self.my_socket.sendall(send_dat.encode(self.code_mode))
elif type(data) == str:
self.my_socket.sendall(data.encode(self.code_mode))
else:
continue
# 如果是退出 直接推出程序
if data == "exit":
break
def main():
# 服务器IP和端口
ip = "127.0.0.1"
port = 12345
my_socket = TcpClient(ip, port)
my_socket.run()
if __name__ == "__main__":
main()
\ No newline at end of file
measurement/oaiflowclassification/multithread_server.py 0 → 100644
View file @ fe3aad70
from socket import *
from threading import Thread
from pool import *
import json
import struct
class TcpServer(object):
"""Tcp服务器"""
def __init__(self, Port):
"""初始化对象"""
self.code_mode = "utf-8" #收发数据编码/解码格式
self.server_socket = socket(AF_INET, SOCK_STREAM) #创建socket
self.server_socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, True) #设置端口复用
self.server_socket.bind((SEVER_HOST, Port)) #绑定IP和Port
self.server_socket.listen(100) #设置为被动socket
print("服务器正在监听...")
def run(self):
"""运行"""
while True:
# 判断程序是否结束
"""
mutex.acquire()
if len(STOP_CLASSIFY) == 1:
print("程序结束")
mutex.release()
break
mutex.release()
"""
client_socket, client_addr = self.server_socket.accept() #等待客户端连接
print("{} 已上线".format(client_addr))
#创建线程为客户端服务
tr = Thread(target=self.recv_data, args=(client_socket, client_addr))
# if !tr.is_alive():
tr.start() #开启线程
self.server_socket.close()
def recv_data(self, client_socket, client_addr):
"""收发数据"""
while True:
# 判断程序是否结束
"""
mutex.acquire()
if len(STOP_CLASSIFY) == 1:
print("程序结束")
mutex.release()
break
mutex.release()
"""
recv = client_socket.recv(80)
# a = []
data = []
if recv:
# print('服务端收到客户端发来的消息:%s' % (recv))
recv = recv[:77]
a = struct.unpack('2i2f1i2l2d1l13B', recv)
print(a)
data = [[], []]
idStr = ""
mystr = a[10:]
for i in range(10):
data[0].append(a[i]+0.0)
for i in range(8):
if i % 4 == 3:
idStr += str(mystr[i])+" "
else:
idStr += str(mystr[i]) + "."
idStr += str(int(mystr[8])*256+int(mystr[9])) + " "
idStr += str(int(mystr[10])*256+int(mystr[11])) + " "
"""
if int(mystr[12]) == 6:
idStr += "TCP"
else:
idStr += "UDP"
"""
idStr += str(int(mystr[12])) +""
data[1].append(idStr)
#data = client_socket.recv(BUFFER_SIZE).decode(self.code_mode)
#if len(data) > 8:
# data = json.loads(data)
if data:
# 关闭连接的客户端的线程
if data == "exit":
# 发送关闭客户端的命令
client_socket.send("close connecting".encode(self.code_mode))
print("{} 已下线".format(client_addr))
break
# 如果接收的命令是close 修改标志位 停止程序
elif data == "close":
mutex.acquire()
close_program()
mutex.release()
break
# 如果发送的命令是 send 则将分类结果发送过来
elif data == "send":
mutex.acquire()
if CLASSIFY_RESULT:
for res in CLASSIFY_RESULT:
client_socket.send(str(res).encode(self.code_mode))
CLASSIFY_RESULT.clear()
else:
client_socket.send("暂无分类结果".encode(self.code_mode))
mutex.release()
else:
# 将数据保存在PREDECT_FLOWS中 加锁
mutex.acquire()
if data not in PREDECT_FLOWS:
PREDECT_FLOWS.append(data)
mutex.release()
print("{}:发送特征的五元组为{}".format(client_addr, data[1]))
# print("缓冲池中有: ", len(PREDECT_FLOWS), " 个待分类的特征，请分类")
# client_socket.send(data.encode(self.code_mode))
else:
#客户端断开连接
print("{} 已下线".format(client_addr))
break
client_socket.close()
def boot_server():
# print("\033c", end="") #清屏
# port = int(input("请输入要绑定的Port:"))
my_server = TcpServer(SEVER_PORT)
my_server.run()
if __name__ == "__main__":
boot_server()
measurement/oaiflowclassification/oai_client/multithread_client.py 0 → 100644
View file @ fe3aad70
from socket import *
from threading import Thread
from random import choice
import json
import time
BUFFER_SIZE = 8192
# 随机发送数据测试的库:
data_list = [
[[221.0, 1350.0, 640.0, 376.26798960315506, 543.0, 21257877.349853516,
4793407917.022705, 1263437211.5135193, 2039103758.0566826, 119541525.84075928],["1"]],
[[171.0, 1460.0, 888.4, 514.1558518581695, 853.0, 11920.928955078125,
17442424058.914185, 4385495722.293854, 7538530505.708111, 49773454.666137695],["2"]],
[[498.0, 1460.0, 1088.8, 455.13356281425786, 1460.0, 36954.87976074219,
43512821.197509766, 11164724.826812744, 18679961.749486398, 554561.6149902344],["3"]],
[[498.0, 1460.0, 1088.8, 455.13356281425786, 1460.0, 30994.415283203125,
44764995.57495117, 16246497.631072998, 18385686.144529935, 10095000.267028809],["4"]],
[[566.0, 1460.0, 1281.2, 357.59999999999997, 1460.0, 22172.927856445312,
21413803.100585938, 5526959.8960876465, 9175244.715715846, 335931.77795410156],["5"]],
[[69.0, 629.0, 246.6, 205.1522361564699, 172.0, 190973.28186035156,
457492113.1134033, 219977498.0545044, 220039359.48775682, 211113452.91137695],["6"]],
[[69.0, 278.0, 137.8, 78.91108920804477, 85.0, 144958.49609375,
462785959.2437744, 210317969.3222046, 211827960.3342278, 189170479.7744751],["7"]],
[[302.0, 1460.0, 807.6, 537.1113850962387, 498.0, 12874.603271484375,
205285072.32666016, 56649982.92922974, 86251862.02980827, 10650992.393493652],["8"]],
[[266.0, 1460.0, 892.8, 490.89730086852177, 780.0, 24080.276489257812,
301223039.6270752, 76725006.10351562, 129634012.70485088, 2826452.2552490234],["9"]],
[[514.0, 1460.0, 1191.6, 371.89762032043177, 1460.0, 10013.580322265625,
890016.5557861328, 236511.23046875, 377434.79666208074, 23007.39288330078],["10"]],
"hhh", "exit", "send", "send"]
class TcpClient(object):
"""Tcp客户端"""
def __init__(self, IP="127.0.0.1", Port=5002):
"""初始化对象"""
self.code_mode = "utf-8" #收发数据编码/解码格式
self.IP = IP
self.Port = Port
self.my_socket = socket(AF_INET, SOCK_STREAM) #创建socket
def run(self):
"""启动"""
self.my_socket.connect((self.IP, self.Port)) #连接服务器
tr = Thread(target=self.recv_data) #创建线程收数据
ts = Thread(target=self.send_data) #创建线程发数据
tr.start() #开启线程
ts.start()
def recv_data(self):
"""收数据"""
while True:
data = self.my_socket.recv(BUFFER_SIZE).decode(self.code_mode)
if data:
if data == "close connecting":
print("已下线")
break
else:
print("{}".format(data))
else:
break
self.my_socket.close()
def send_data(self):
"""发数据"""
while True:
# 接收数据
data = choice(data_list)
if type(data) == list:
print("已发送五元组{}的特征".format(data[1][0]))
else:
print("发送命令: ", data)
# 两s发送一次 测试
time.sleep(2)
# 判断data的类型 如果是列表用 json传输 字符串就直接传输
if type(data) == list:
send_dat = json.dumps(data)
self.my_socket.sendall(send_dat.encode(self.code_mode))
elif type(data) == str:
self.my_socket.sendall(data.encode(self.code_mode))
else:
continue
# 如果是退出 直接推出程序
if data == "exit":
break
def main():
# 服务器IP和端口
ip = "192.168.136.137"
port = 5005
my_socket = TcpClient(ip, port)
my_socket.run()
if __name__ == "__main__":
main()
\ No newline at end of file
measurement/oaiflowclassification/path_utils.py 0 → 100644
View file @ fe3aad70
from pathlib import Path
import os.path
def get_prj_root():
return os.path.abspath(os.curdir)
# return Path(__file__).parent
if __name__ == '__main__':
print(get_prj_root())
measurement/oaiflowclassification/pool.py 0 → 100644
View file @ fe3aad70
import threading
"""
该文件定义使用的常量 和 进程锁
"""
# 设置服务器的IP和端口
SEVER_HOST = "127.0.0.1" # 发送到windows测试
SEVER_PORT = 12345
# 缓冲区大小
BUFFER_SIZE = 8192
# 共享的全局变量 保存解析的特征 用于客户端发送和服务器端读取的缓冲区
PREDECT_FLOWS = []
# 共享的全局变量 保存分类的结果
CLASSIFY_RESULT = []
# 是否停止分类的标志位 长度为1退出程序
STOP_CLASSIFY = []
# 定义修改全局变量的函数
def close_program():
global STOP_CLASSIFY
STOP_CLASSIFY.append(1)
# 创建全局的进程锁
mutex = threading.Lock()
# 数据库操作
# 数据库地址
DB_ADDR = '127.0.0.1'
# 数据库用户名
USER_NAME = 'root'
# 数据库用户密码
USER_PASSWORD = '123456'
# 数据库名称
DB_NAME = 'mytestdb'
# 测试使用的数据
test = [[221.0, 1350.0, 640.0, 376.26798960315506, 543.0,
21257877.349853516, 4793407917.022705, 1263437211.5135193,
2039103758.0566826, 119541525.84075928],["sip sport dip dport protocol"]]
measurement/oaiflowclassification/predict.py 0 → 100644
View file @ fe3aad70
"""
预测函数：随机森林
每次运行前，检查：
四个需要修改的地方，命名是否正确
最后的运行模式是否正确
"""
from common_utils import * # 修改了
from argparse import ArgumentParser
from collections import namedtuple
from typing import List, Dict
from datetime import datetime
from path_utils import get_prj_root
import numpy as np
# from sklearn.externals import joblib
import joblib
from sklearn.ensemble import RandomForestClassifier # 训练模型
random.seed(datetime.now())
model_dir = os.path.join(get_prj_root(), "classify/models") # 修改：模型model文件夹路径
predict_model_pkl = os.path.join(model_dir, "dt2.pkl") # 修改：模型的版本，只用修改此处就行
Instance = namedtuple("Instance", ["features", "label"]) # 实例
dirs = {
"video": "./tmp/dt/video",
"iot": "./tmp/dt/iot",
"voip": "./tmp/dt/voip",
"AR": "./tmp/dt/AR",
}
instances_dir = os.path.join(get_prj_root(), "classify/instances") # 修改：instances路径
def train_and_predict():
iot = load_pkl(os.path.join(instances_dir, "iot.pkl")) # 不同实例的pkl是不同特征的
videos = load_pkl(os.path.join(instances_dir, "video.pkl"))
voip = load_pkl(os.path.join(instances_dir, "voip.pkl"))
AR = load_pkl(os.path.join(instances_dir, "AR.pkl"))
for i in videos:
assert i.label == 0
for i in iot:
assert i.label == 1
for i in voip:
assert i.label == 2
for i in AR:
assert i.label == 3
# print(videos)
debug("# iot instances {}".format(len(iot)))
debug("# video instances {}".format(len(videos)))
debug("# VOIP instances {}".format(len(voip)))
debug("# AR instances {}".format(len(AR)))
random.shuffle(voip) # 打乱排序
random.shuffle(iot)
random.shuffle(videos)
random.shuffle(AR)
n_video_train = int(len(videos) * 0.7)
n_video_test = len(videos) - n_video_train
video_train = videos[:n_video_train]
video_test = videos[n_video_train:]
iot_train = iot[:n_video_train]
iot_test = iot[len(iot) - len(video_test):]
voip_train = voip[:n_video_train]
voip_test = voip[len(voip) - len(video_test):]
AR_train = AR[:n_video_train]
AR_test = AR[len(AR) - len(video_test):]
info("#video train {}".format(len(video_train)))
info("#iot train {}".format(len(iot_train)))
info("#voip train {}".format(len(voip_train)))
info("#AR train {}".format(len(AR_train)))
train = []
train.extend(iot_train)
train.extend(video_train)
train.extend(voip_train)
train.extend(AR_train)
random.shuffle(train)
train_x = [x.features for x in train]
train_y = [x.label for x in train]
# test 1:1
test = []
info("#video test {}".format(len(video_test)))
info("#iot test {}".format(len(iot_test)))
info("#voip test {}".format(len(voip_test)))
info("#AR test {}".format(len(AR_test)))
test.extend(video_test)
test.extend(iot_test)
test.extend(voip_test)
test.extend(AR_test)
random.shuffle(test)
test_x = [t.features for t in test]
test_y = [t.label for t in test]
# 训练以及预测
predict_model = RandomForestClassifier(oob_score=True) # 引入训练方法
predict_model.fit(train_x, train_y) # 队训练数据进行拟合
predicts = predict_model.predict(test_x)
"""
dt = DT()
dt.fit((train_x, train_y))
predicts = dt.predict(test_x)
"""
"""
# 打印预测的结果
print(predicts)
print("-------------------------------")
"""
# 保存模型
fn_name = os.path.join(model_dir, predict_model_pkl)
joblib.dump(predict_model, predict_model_pkl)
# 评价模型
count = 0
for idx in range(len(test_x)):
if int(predicts[idx]) == int(test_y[idx]):
count += 1
# print(count / len(test_x))
return count / len(test_x)
# save_pkl(predict_model_pkl, knn.model)
# knn.save_model(dt_model_pkl) #储存模型
if __name__ == '__main__':
n = 10
s = 0
predict_sum = 0
for i in range(n):
s = train_and_predict()
predict_sum = predict_sum + s
print(predict_sum / n)
measurement/oaiflowclassification/process_bar.py 0 → 100644
View file @ fe3aad70
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 18-5-21 下午3:44
# @Author : LK
# @File : 进度条.py
# @Software: PyCharm
import sys
import time
def process_bar(precent, width=50):
use_num = int(precent*width)
space_num = int(width-use_num)
precent = precent*100
# 第一个和最后一个一样梯形显示, 中间两个正确,但是在python2中报错
#
# print('[%s%s]%d%%'%(use_num*'#', space_num*' ',precent))
# print('[%s%s]%d%%'%(use_num*'#', space_num*' ',precent), end='\r')
print('[%s%s]%d%%'%(use_num*'#', space_num*' ',precent),file=sys.stdout,flush=True, end='\r')
# print('[%s%s]%d%%'%(use_num*'#', space_num*' ',precent),file=sys.stdout,flush=True)
# for i in range(21):
# precent = i/20
# process_bar(precent)
# time.sleep(0.2)
measurement/oaiflowclassification/requirement.txt 0 → 100644
View file @ fe3aad70
loguru
numpy
joblib
scikit-learn==0.24.1
pymysql
\ No newline at end of file
measurement/oaiflowclassification/save_result.py 0 → 100644
View file @ fe3aad70
"""
该文件用于将分类好的结果保存到mysql数据库中
"""
import pymysql
from pool import *
# 建立连接
def connect_database(addr, uer_name, usr_paswd, db_name):
return pymysql.connect(host = addr, user = uer_name, passwd = usr_paswd, database = db_name, charset='utf8' )
# 插入数据
def mysqldb_insert(con, srcIP, dstIP, srcPort, dstPort, protocol,flowType):
# con = connect_database(DB_ADDR, USER_NAME, USER_PASSWORD, DB_NAME)
cursor = con.cursor()
sql = "INSERT INTO measure(srcIP, dstIP, srcPort, dstPort, protocol, flowType) VALUES ({}, {}, {}, {}, {},\'{}\') ON DUPLICATE KEY UPDATE flowType = \'{}\'".format(srcIP, dstIP, srcPort, dstPort, protocol, flowType,flowType)
# print(sql)
try:
# 执行sql语句
cursor.execute(sql)
# 提交到数据库执行
con.commit()
except:
# 发生错误时回滚
con.rollback()
#cursor.close()
#con.close()
print("插入数据成功！")
# 查询数据
def mysqldb_research():
return
# 清空数据
def mysqldb_clearall(con):
cursor = con.cursor()
sql = "TRUNCATE TABLE measure;"
try:
# 执行sql语句
cursor.execute(sql)
# 提交到数据库执行
con.commit()
except:
# 发生错误时回滚
con.rollback()
print("表已清空！")
if __name__ == "__main__":
# write_database()
con = connect_database(DB_ADDR, USER_NAME, USER_PASSWORD, DB_NAME)
mysqldb_insert(con, 3,4,1,1,'UDP',"iot")
mysqldb_insert(con, 1,2,1,1,1,"video")
mysqldb_insert(con, 1,3,3,1,1,"video")
mysqldb_insert(con, 1,4,3,1,1,"video")
# mysqldb_clearall(con)
measurement/oaiflowclassification/scripts/init.sh 0 → 100755
View file @ fe3aad70
#!/usr/bin/env bash
# Shell script absolute path
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
echo "当前脚本所在路径为: $DIR"
cd $DIR
cd ..
# Check Python3
python3_version=$(python3 -V|awk '{print $2}')
if [ $python3_version ]; then
echo "python3已安装,当前版本为:{$python3_version}"
else
apt update && apt install -y python3
fi
# Check Python3-venv
python3venv_version=$(dpkg --status python3-venv 2>/dev/null|grep Version|awk '{print $2}')
if [ $python3venv_version ]; then
echo "python3-venv已安装,当前版本为:{$python3venv_version}"
else
apt update && apt install -y python3-venv
fi
# Check if .py38 has been built
py38=$(ls -a ../|grep .py38)
if [ $py38 ]; then
echo "Python3虚拟环境目录已安装"
else
python3 -m venv .py38
fi
# .py38 install requirements
source ./.py38/bin/activate
pip install -r ./requirement.txt
\ No newline at end of file
measurement/oaiflowclassification/scripts/run.sh 0 → 100755
View file @ fe3aad70
#!/usr/bin/env bash
# Shell script absolute path
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
echo "当前脚本所在路径为: $DIR"
cd $DIR
cd ..
# Check if .py38 has been built
py38=$(ls -a ./|grep .py38)
if [ $py38 ]; then
echo "Python3虚拟环境目录已安装"
else
echo "请先运行init.sh创建Python3虚拟环境并下载依赖"
exit
fi
# Enter Python3 virtual environment
source ./.py38/bin/activate
python ./main.py
\ No newline at end of file
measurement/oaiflowclassification/test.txt 0 → 100644
View file @ fe3aad70
This diff is collapsed.
measurement/oaiflowclassification/train.py 0 → 100644
View file @ fe3aad70
"""
10个特征的（前五个包大小，后五个包间隔）：最小值，最大值，平均值，方差，中位数
每次运行前，检查：
四个需要修改的地方，命名是否正确
最后的运行模式是否正确
这个文件用于把pcap解析的文件生成特征和标签的形式 并且5个包一组
"""
from common_utils import * # 修改了
from argparse import ArgumentParser
from collections import namedtuple
from typing import List, Dict
from path_utils import get_prj_root
from model import DT # 修改了
from datetime import datetime
from path_utils import get_prj_root
import numpy as np
from sklearn.decomposition import PCA
import time
# start counting time
start = time.time()
random.seed(datetime.now())
# get the path of the models
model_dir = os.path.join(get_prj_root(), "classify/models") # 修改：模型models路径
dt_model_pkl = os.path.join(model_dir, "dt2_9.pkl") # 修改：模型的版本，只用修改此处就行
Instance = namedtuple("Instance", ["features", "label"]) # 实例
win_size = 5 # 窗口大小
limit = 100000
# the path
# iot-物联网流 video-视频流 voip-音频流 AR-AR流用的高清视频流代替
dirs = {
"video": "./tmp/dt/video",
"iot": "./tmp/dt/iot",
"voip": "./tmp/dt/voip",
"AR": "./tmp/dt/AR",
}
instances_dir = os.path.join(get_prj_root(), "classify/instances") # 修改：instances路径
# 获取特征
def get_median(data): # 产生中位数
data.sort()
half = len(data) // 2
return (data[half] + data[~half]) / 2
def gen_single_instance(dir_name, flow, flow_type):
# debug("generate {}".format(flow["file"]))
def extract_features(raw_features: List[float]): # 修改特征
extracted_features = []
raw_features = [r for r in raw_features if int(r) >= 0]
extracted_features.append(min(raw_features))
extracted_features.append(max(raw_features))
extracted_features.append(sum(raw_features) / len(raw_features))
extracted_features.append(np.std(raw_features)) # 标准差
extracted_features.append(get_median(raw_features)) # 中位数
return extracted_features
features = []
idts = []
ps = []
idt_file = os.path.join(dir_name, flow["idt"]) # 包大小
ps_file = os.path.join(dir_name, flow["ps"]) # 包间隔
with open(idt_file, 'r') as fp:
lines = fp.readlines()
fp.close()
lines = [l.strip() for l in lines] # .strip()用于移除字符串头尾指定的字符（默认为空格或换行符）或字符序列。
lines = [l for l in lines if len(l) > -1]
if len(lines) > win_size:
lines = lines[:win_size]
for l in lines:
idts.append(float(l))
with open(ps_file, "r") as fp:
lines = fp.readlines()
fp.close()
lines = [l.strip() for l in lines]
lines = [l for l in lines if len(l) > 0]
if len(lines) > win_size:
lines = lines[:win_size]
for l in lines:
ps.append(float(l))
# 有很奇怪的现象
ps = [p for p in ps if p > 0]
if len(ps) == 0:
print(flow["ps"])
return None
idts = [i for i in idts if i >= 0]
if len(idts) == 0:
return None
features.extend(extract_features(ps)) # 包间隔的数理统计
features.extend(extract_features(idts)) # 包大小的数理统计
if flow_type == "video":
label = 0
elif flow_type == "iot":
label = 1
elif flow_type == "voip":
label = 2
elif flow_type == "AR":
label = 3
else:
err("Unsupported flow type")
raise Exception("Unsupported flow type")
return Instance(features=features, label=label)
def generate():
instances_dir = os.path.join(get_prj_root(), "classify/instances") # 修改：instances_dir实例的路径
for flow_type, dirname in dirs.items():
stats_fn = os.path.join(dirname, "statistics.json") # statistics.json流量统计的文件
debug(stats_fn)
statistics = load_json(os.path.join(dirname, "statistics.json"))
debug("#flows {}".format(statistics["count"]))
flows: List = statistics["flows"]
sorted(flows, key=lambda f: -f["num_pkt"])
if len(flows) > limit:
flows = flows[:limit]
instances = [gen_single_instance(dirname, f, flow_type) for f in flows]
instances = [i for i in instances if i is not None]
debug("#{} instances {}".format(flow_type, len(instances)))
# print(len(instances))
save_pkl(os.path.join(instances_dir, "{}.pkl".format(flow_type)), instances) # 保存Python内存数据到文件
if __name__ == '__main__':
parser = ArgumentParser()
print("running mode\n"
"1. generate instances\n"
"2. train dt\n")
parser.add_argument("--mode", type=int, default=1) # default为模式修改
args = parser.parse_args()
mode = int(args.mode)
if mode == 1:
generate()
end = time.time()
print("程序运行时间:%.2f秒" % (end - start))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment